Showing preview only (1,233K chars total). Download the full file or copy to clipboard to get everything.
Repository: Aquila-Network/AquilaDB
Branch: main
Commit: b086fa65f5cc
Files: 273
Total size: 1.1 MB
Directory structure:
gitextract_6lg6_8cc/
├── README.md
└── aquila/
├── encoder/
│ ├── CODE_OF_CONDUCT.md
│ ├── Dockerfile
│ ├── LICENSE
│ ├── README.md
│ ├── authentication.py
│ ├── config.yml
│ ├── encoder.py
│ ├── index.py
│ ├── ipfs.service
│ ├── manager.py
│ ├── requirements.txt
│ ├── run_tests.sh
│ ├── test/
│ │ ├── __init__.py
│ │ └── apis/
│ │ └── hub_fns.py
│ ├── test.py
│ └── utils/
│ ├── CID.py
│ ├── config.py
│ ├── cryptops.py
│ ├── downloader.py
│ └── schema.py
├── metricstore/
│ ├── .travis.yml
│ ├── CODE_OF_CONDUCT.md
│ ├── DB_config.yml
│ ├── Dockerfile
│ ├── DockerfileBig
│ ├── LICENSE
│ ├── README.md
│ ├── authentication.py
│ ├── index.py
│ ├── install.sh
│ ├── manager.py
│ ├── router.py
│ ├── run_tests.sh
│ ├── test/
│ │ ├── __init__.py
│ │ └── apis/
│ │ ├── auth_fns.py
│ │ ├── db_fns.py
│ │ ├── doc_fns.py
│ │ └── search_fns.py
│ ├── utils/
│ │ ├── CID.py
│ │ ├── config.py
│ │ ├── cryptops.py
│ │ └── schema.py
│ ├── vec_index/
│ │ ├── hannoy.py
│ │ └── hfaiss.py
│ └── wal/
│ ├── walman.py
│ └── walmon.py
├── network/
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── search/
│ ├── .dockerignore
│ ├── .nvmrc
│ ├── Dockerfile
│ ├── README.md
│ ├── package.json
│ ├── src/
│ │ ├── @types/
│ │ │ ├── express/
│ │ │ │ └── index.d.ts
│ │ │ └── index.d.ts
│ │ ├── app.ts
│ │ ├── config/
│ │ │ ├── db.ts
│ │ │ └── redisConnection.ts
│ │ ├── controller/
│ │ │ ├── AuthController.ts
│ │ │ ├── BookmarkController.ts
│ │ │ ├── CollectionController.ts
│ │ │ ├── CollectionSubscriptionController.ts
│ │ │ ├── CustomerController.ts
│ │ │ ├── HomeController.ts
│ │ │ └── dto/
│ │ │ ├── AuthControllerDto.ts
│ │ │ ├── BookmarkControllerDto.ts
│ │ │ ├── CollectionControllerDto.ts
│ │ │ ├── CollectionSubscriptionControllerDto.ts
│ │ │ └── CustomerControllerDto.ts
│ │ ├── entity/
│ │ │ ├── Bookmark.ts
│ │ │ ├── BookmarkPara.ts
│ │ │ ├── BookmarkParaTemp.ts
│ │ │ ├── BookmarkTemp.ts
│ │ │ ├── Collection.ts
│ │ │ ├── CollectionSubscription.ts
│ │ │ ├── CollectionSubscriptionTemp.ts
│ │ │ ├── CollectionTemp.ts
│ │ │ ├── Customer.ts
│ │ │ └── CustomerTemp.ts
│ │ ├── helper/
│ │ │ ├── auth.ts
│ │ │ ├── decorators/
│ │ │ │ └── jwtPayloadData.ts
│ │ │ └── user.ts
│ │ ├── index.ts
│ │ ├── job/
│ │ │ ├── AppQueue.ts
│ │ │ ├── appWorker.ts
│ │ │ ├── appWorkerProcessor.ts
│ │ │ └── types.ts
│ │ ├── lib/
│ │ │ ├── AquilaClientService.ts
│ │ │ └── ConfigService.ts
│ │ ├── middleware/
│ │ │ ├── global/
│ │ │ │ ├── AuthMiddleware.ts
│ │ │ │ ├── GlobalErrorMiddleware.ts
│ │ │ │ └── TokenParserMiddleware.ts
│ │ │ └── validator/
│ │ │ ├── bookmark/
│ │ │ │ ├── AddBookmarkValidator.ts
│ │ │ │ ├── GetBookmarkByCollectionIdValidator.ts
│ │ │ │ ├── GetFeaturedBookmarkValidator.ts
│ │ │ │ ├── GetPublicBookmarkByCollectionIdParamValidator.ts
│ │ │ │ └── GetPublicBookmarkByCollectionIdValidator.ts
│ │ │ ├── collection/
│ │ │ │ ├── GetAllFeaturedCollectionValidator.ts
│ │ │ │ ├── GetAllPublicCollectionValidator.ts
│ │ │ │ └── GetPublicCollectionByIdValidator.ts
│ │ │ ├── csubscription/
│ │ │ │ ├── SubscribeCollectionValidator.ts
│ │ │ │ └── UnSubscribeCollectionValidator.ts
│ │ │ └── customer/
│ │ │ ├── ActivateCustomerValidator.ts
│ │ │ ├── CreateCustomerValidator.ts
│ │ │ ├── GetCustomerPublicInfoByIdValidator.ts
│ │ │ └── UpdateCustomerValidator.ts
│ │ ├── migrations/
│ │ │ └── 1676714378990-bookmark.ts
│ │ ├── service/
│ │ │ ├── AuthService.ts
│ │ │ ├── BookmarkService.ts
│ │ │ ├── CollectionService.ts
│ │ │ ├── CollectionSubscriptionService.ts
│ │ │ ├── CustomerService.ts
│ │ │ └── dto/
│ │ │ ├── AuthServiceDto.ts
│ │ │ ├── BookmarkServiceDto.ts
│ │ │ ├── CollectionServiceDto.ts
│ │ │ ├── CollectionSubscriptionServiceDto.ts
│ │ │ └── CustomerServiceDto.ts
│ │ └── utils/
│ │ ├── errors/
│ │ │ └── ValidationError.ts
│ │ ├── randomAnimals.ts
│ │ └── validate.ts
│ └── tsconfig.json
├── txt_transform/
│ ├── Dockerfile_mercury
│ ├── Dockerfile_txtpick
│ ├── app.py
│ ├── package.json
│ ├── requirements.txt
│ ├── server.js
│ ├── test.html
│ └── test.py
└── view/
├── .dockerignore
├── .eslintrc.json
├── @types/
│ └── next-auth.d.ts
├── Dockerfile
├── README.md
├── components/
│ ├── hoc/
│ │ └── InitComponent.tsx
│ ├── layout/
│ │ ├── base/
│ │ │ └── baseLayout.tsx
│ │ ├── boxCenter/
│ │ │ ├── BoxCenterLayout.module.scss
│ │ │ └── BoxCenterLayout.tsx
│ │ ├── childLayout/
│ │ │ └── settings/
│ │ │ ├── Header.module.scss
│ │ │ ├── Header.tsx
│ │ │ ├── SettingsLayout.module.scss
│ │ │ ├── SettingsLayout.tsx
│ │ │ ├── Sidebar.module.scss
│ │ │ └── Sidebar.tsx
│ │ └── main/
│ │ ├── AddLink.module.scss
│ │ ├── AddLink.tsx
│ │ ├── Footer.module.scss
│ │ ├── Footer.tsx
│ │ ├── Header.module.scss
│ │ ├── Header.tsx
│ │ ├── MainLayout.module.scss
│ │ └── MainLayout.tsx
│ ├── pages/
│ │ ├── account/
│ │ │ └── editProfile/
│ │ │ ├── EditProfileForm.module.scss
│ │ │ ├── EditProfileForm.tsx
│ │ │ ├── EditProfileHeader.module.scss
│ │ │ ├── EditProfileHeader.tsx
│ │ │ ├── EditProfileWrapper.module.scss
│ │ │ └── EditProfileWrapper.tsx
│ │ ├── collection/
│ │ │ └── bookmark/
│ │ │ └── CollectionBookmarks/
│ │ │ ├── CollectionBookmarksPageWrapper.module.scss
│ │ │ ├── CollectionBookmarksPageWrapper.tsx
│ │ │ ├── SearchBar.module.scss
│ │ │ ├── SearchBar.tsx
│ │ │ ├── SearchPageProfile.module.scss
│ │ │ ├── SearchPageProfile.tsx
│ │ │ ├── SearchResultItem.module.scss
│ │ │ ├── SearchResultItem.tsx
│ │ │ ├── SearchResults.module.scss
│ │ │ └── SearchResults.tsx
│ │ ├── explore/
│ │ │ ├── Explore.module.scss
│ │ │ ├── Explore.tsx
│ │ │ ├── ExploreCategoryItem.module.scss
│ │ │ ├── ExploreCategoryItem.tsx
│ │ │ ├── ExploreCategoryList.module.scss
│ │ │ ├── ExploreCategoryList.tsx
│ │ │ └── ExplorePageWrapper.tsx
│ │ ├── home/
│ │ │ ├── HomePageWrapper.module.scss
│ │ │ ├── HomePageWrapper.tsx
│ │ │ ├── SearchBar.module.scss
│ │ │ ├── SearchBar.tsx
│ │ │ ├── SearchPageProfile.module.scss
│ │ │ ├── SearchPageProfile.tsx
│ │ │ ├── SearchResultItem.module.scss
│ │ │ ├── SearchResultItem.tsx
│ │ │ ├── SearchResults.module.scss
│ │ │ └── SearchResults.tsx
│ │ ├── index/
│ │ │ ├── AllControl.module.scss
│ │ │ ├── AllControl.tsx
│ │ │ ├── Discover.module.scss
│ │ │ ├── Discover.tsx
│ │ │ ├── Hero.module.scss
│ │ │ ├── Hero.tsx
│ │ │ ├── IndexPageWrapper.tsx
│ │ │ ├── Story.module.scss
│ │ │ └── Story.tsx
│ │ ├── signIn/
│ │ │ ├── SignInForm.module.scss
│ │ │ ├── SignInForm.tsx
│ │ │ └── SignInPageWrapper.tsx
│ │ ├── signUp/
│ │ │ ├── SecretKey.module.scss
│ │ │ ├── SecretKey.tsx
│ │ │ ├── SignUpForm.module.scss
│ │ │ ├── SignUpForm.tsx
│ │ │ └── SignUpPageWrapper.tsx
│ │ └── subscription/
│ │ ├── Collection.module.scss
│ │ ├── Collection.tsx
│ │ ├── HomePageWrapper.tsx
│ │ ├── SearchBar.module.scss
│ │ ├── SearchBar.tsx
│ │ ├── SearchPageProfile.module.scss
│ │ ├── SearchPageProfile.tsx
│ │ ├── SearchResultItem.module.scss
│ │ ├── SearchResultItem.tsx
│ │ ├── SearchResults.module.scss
│ │ ├── SearchResults.tsx
│ │ ├── SubscribedCollections.module.scss
│ │ ├── SubscribedCollections.tsx
│ │ ├── SubscriptionPageWrapper.module.scss
│ │ └── SubscriptionPageWrapper.tsx
│ └── ui/
│ ├── alert/
│ │ ├── Alert.module.scss
│ │ └── Alert.tsx
│ ├── layout/
│ │ ├── Container.module.scss
│ │ └── Container.tsx
│ ├── modal/
│ │ ├── Modal.module.scss
│ │ └── Modal.tsx
│ └── progressLoader/
│ ├── ProgressLoader.module.scss
│ └── ProgressLoader.tsx
├── middleware.ts
├── next.config.js
├── package.json
├── pages/
│ ├── _app.tsx
│ ├── _document.tsx
│ ├── account/
│ │ └── edit-profile.tsx
│ ├── api/
│ │ ├── auth/
│ │ │ └── [...nextauth].ts
│ │ └── hello.ts
│ ├── collection/
│ │ └── bookmark/
│ │ └── [collectionId].tsx
│ ├── explore.tsx
│ ├── home.tsx
│ ├── index.tsx
│ ├── sign-in.tsx
│ ├── sign-up.tsx
│ ├── subscription.tsx
│ └── test.tsx
├── store/
│ ├── index.ts
│ └── slices/
│ ├── auth.ts
│ ├── bookmark/
│ │ ├── addLink.ts
│ │ ├── getLoggedInCustBookmarksByCollectionId.ts
│ │ └── getPublicBookmarksByCollectionId.ts
│ ├── collection/
│ │ ├── getAllPublicCollections.ts
│ │ ├── getCollectionById.ts
│ │ ├── getCustomerSubscriptions.ts
│ │ ├── getFeaturedCollections.ts
│ │ ├── getLoggedInCustCollections.ts
│ │ ├── getSubscribedCollections.ts
│ │ ├── isCollectionSubscribed.ts
│ │ ├── subscribeCollectionById.ts
│ │ └── unSubscribeCollectionById.ts
│ ├── customer/
│ │ ├── activateCustomer.ts
│ │ ├── getCurrentLoggedInCustomer.ts
│ │ ├── getCustomerById.ts
│ │ └── updateCustomer.ts
│ ├── errors/
│ │ └── AsyncThunkSubmissionError.ts
│ ├── generateName.ts
│ ├── signup.ts
│ ├── types/
│ │ ├── Bookmark.ts
│ │ ├── Collection.ts
│ │ ├── CollectionSubscription.ts
│ │ ├── Customer.ts
│ │ └── validationErrors.ts
│ └── utils/
│ └── createError.ts
├── styles/
│ └── globals.scss
├── tsconfig.json
└── utils/
└── api.ts
================================================
FILE CONTENTS
================================================
================================================
FILE: README.md
================================================
<div align="center">
<a href="https://aquila.network">
<img
src="https://user-images.githubusercontent.com/19545678/133918727-5a37c6be-676f-427b-8c86-dd50f58d1287.png"
alt="Aquila Network Logo"
height="64"
/>
</a>
<br />
<p>
<h3>
<b>
Aquila DB
</b>
</h3>
</p>
<p>
<b>
Easy to use Neural Search Engine
</b>
</p>
<br/>
</div>
**[Aquila DB](https://github.com/Aquila-Network/AquilaDB)** is a Neural search engine. In other words, it is a database to index **Latent Vectors** generated by ML models along with **JSON Metadata** to perform **k-NN** retrieval. It is dead simple to set up, language-agnostic, and drop in addition to your Machine Learning Applications. Aquila DB, as of current features is a ready solution for Machine Learning engineers and Data scientists to build **[Neural Information Retrieval](https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf)** applications out of the box with minimal dependencies.
> This project is still in alpha version & we're already using it in production to power semantic search at https://aquila.network.
Wanna support this project? Yes, we love getting a **star** ⭐ and **shout-out** 🗣️ 🤗
Join [Community chat and get support: ](https://discord.gg/5YP7zHS)
# Who is this for
* If you are working on a data science project and need to store a hell of a lot of data and retrieve similar data based on some feature vector, this will be a useful tool to you, with extra benefits a real world web application needs.
* Are you dealing with a lot of images and related metadata? Want to find similar ones? You are at the right place.
* If you are looking for a document database, this is not the right place for you.
# Technology
Aquila DB powers search features of Aquila Network. Here is where Aquila DB fits in the entire ecosystem:
<div align="center">
<img
src="https://user-images.githubusercontent.com/19545678/133918438-997af8ec-67ef-4f7c-95ab-16d2f28e79e3.png"
alt="Aquila DB Architecture"
height="400"
/>
<br/>
</div>
If you are serious and wanna dive down the rabbit hole, read our **[whitepapers](https://github.com/Aquila-Network/whitepaper)** and **[technical specifications](https://github.com/Aquila-Network/specs)** (being actively worked on).
**As a side note**, everything in **[Aquila Network](https://github.com/Aquila-Network)** is defined by the specifications and a large chunk of our efforts goes into it. We also maintain quality implementations of those specifications with non-technical users in mind. This is to make sure that - Aquila Network is fully open, decentralized by design, and Fair. You can follow those specifications to implement your alternative software and still interact with the network without any restrictions.
# Install
### Debian
Run `curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/install.sh | /bin/bash -s -- -d 1 `.
### Docker
**You need docker installed in your system**
Build image (lite): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/Dockerfile -t aquiladb:local`
Build image (big data): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/DockerfileBig -t aquiladb:localbg`
Run image (to deploy Aquila DB lite): `docker run -p 5001:5001 -d aquiladb:local`
Run image (to deploy Aquila DB big): `docker run -p 5001:5001 -d aquiladb:localbg`
# Client SDKs
We currently have multiple client libraries in progress to abstract the communication between deployed Aquila DB and your applications.
[Python](https://github.com/Aquila-Network/AquilaPy)
[Node JS](https://github.com/Aquila-Network/AquilaJS)
## Where to get private key (wallet key) for client authentication
When you use a client library to authenticate with AquilaDB, you might need access the same private key (wallet key) used by AquilaDB. This key is located inside `/ossl/` directory within AquilaDB docker container (in your computer if you have installed AquilaDB directly without docker). To access the keys inside your AquilaDB container, follow below steps:
* identify `CONTAINER ID` for the already running `aquiladb` docker instance:
`docker ps`
* take a copy of private keys from docker container to your host machine:
`docker cp CONTAINER_ID:/ossl/ ./`
* now you will see a new directory named `ossl` at your current location. Use the keys inside it.
#### tips for advanced users
If your pipeline requires the private keys to be generated in advance, you can do it in your host machine and then mount it to the container's `/ossl/` directory.
Run:
```
mkdir -p <host>/ossl/
openssl genrsa -passout pass:1234 -des3 -out <host>/ossl/private.pem 2048
openssl rsa -passin pass:1234 -in <host>/ossl/private.pem -outform PEM -pubout -out <host>/ossl/public.pem
openssl rsa -passin pass:1234 -in <host>/ossl/private.pem -out <host>/ossl/private_unencrypted.pem -outform PEM
```
# Progress
This project is still and will be under active development with intermediate production releases. It can either be used as a standalone database or as a participating node in Aquila Network. Please note, [Aquila Port](https://github.com/Aquila-Network/specs/blob/main/README.md#aquila-port) (peer-peer network layer for Aquila DB nodes) is also a work in progress. Currently, you need to deploy your custom models to feed vector embeddings to Aquila DB, until [Aquila Hub](https://github.com/Aquila-Network/specs/blob/main/README.md#aquila-hub) developments get started.
# Contribute
We have [prepared a document](https://docs.google.com/document/d/1bT2_9FQIxQpx_rdYbkTukn_DJRi_haVK_ixTf8uTaDE/edit?usp=sharing) to get anyone interested to contribute, immediately started with Aquila DB.
Here is our high-level [release roadmap](https://user-images.githubusercontent.com/19545678/62313851-5af82880-b4af-11e9-84f6-21e24bf46e8a.png).
# Learn
We have started meeting developers and do small talks on Aquila DB. Here are the slides that we use on those occasions: http://bit.ly/AquilaDB-slides
**Video:**
[<img alt="introduction to Neural Information retrieval with AquilaDB" src="http://img.youtube.com/vi/-VYpjpLXU5Q/0.jpg" width="300" />](http://www.youtube.com/watch?v=-VYpjpLXU5Q)
As of current AquilaDB release features, you can build **[Neural Information Retrieval](https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf)** applications out of the box without any external dependencies. Here are some useful links to learn more about it and start building:
* Microsoft published a paper and youtube video on this to onboard anyone interested:
* paper: https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf
* video: https://www.youtube.com/watch?v=g1Pgo5yTIKg
* Embeddings for Everything: Search in the Neural Network Era: https://www.youtube.com/watch?v=JGHVJXP9NHw
* Autoencoders are one such deep learning algorithms that will help you to build semantic vectors - foundation for Neural Information retrieval. Here are some links to Autoencoders based IR:
* go to chapter 15 in this link: https://www.cs.toronto.edu/~hinton/coursera_lectures.html
* https://www.coursera.org/lecture/ml-foundations/examples-of-document-retrieval-in-action-CW25H
* https://www.coursera.org/lecture/intro-to-deep-learning/autoencoders-101-QqBOa
* Note that, the idea of information retrieval applies not only to text data but for any data. All you need to do is, encode any source datatype to a dense vector with deep neural networks.
<br/><br/>
<h1 align="center">Our Sponsors</h1>
<p align="center"><b></b></p>
<br/>
> email us to sponsor this project [adbadmin@protonmail.ch](mailto:adbadmin@protonmail.ch).
<br/><br/>
# Citing Aquila DB
If you use Aquila DB in an academic paper, we would 😍 to be cited. Here are the two ways of citing Aquila DB:
```
\footnote{https://github.com/Aquila-Network/AquilaDB}
```
```
@misc{AquilaNetwork2019AquilaDB,
title={AquilaDB: Neural Search Engine},
author={Jubin Jose, Nibin Peter},
howpublished={\url{https://github.com/Aquila-Network/AquilaDB}},
year={2019}
}
```
# License
Apache License 2.0 [license file](https://github.com/Aquila-Network/AquilaDB/blob/master/LICENSE)
created by ❤️ with a-mma (a_മ്മ)
================================================
FILE: aquila/encoder/CODE_OF_CONDUCT.md
================================================
# AquilaDB Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, sex characteristics, gender identity and expression,
level of experience, education, socio-economic status, nationality, personal
appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at humans@aquiladb.xyz. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq
================================================
FILE: aquila/encoder/Dockerfile
================================================
# start a new build stage
FROM ubuntu:latest as builder
# set work directory
ENV ROOT_DIR /home/root
WORKDIR $ROOT_DIR
# preperations
ENV PATH="$ROOT_DIR/env/bin:$PATH"
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt update && apt install -y git nano python3.8 python3-pip libssl-dev && \
pip3 install virtualenv
RUN cd $ROOT_DIR && \
mkdir -p ahub && \
cd ahub && \
git clone https://github.com/Aquila-Network/AquilaHub.git . && \
virtualenv $ROOT_DIR/env && \
source $ROOT_DIR/env/bin/activate && \
cd src && pip3 install -r requirements.txt
RUN mkdir -p /ossl/ && \
openssl genrsa -passout pass:1234 -des3 -out /ossl/private.pem 2048 && \
openssl rsa -passin pass:1234 -in /ossl/private.pem -outform PEM -pubout -out /ossl/public.pem && \
openssl rsa -passin pass:1234 -in /ossl/private.pem -out /ossl/private_unencrypted.pem -outform PEM
# install and start demon
RUN mkdir -p /data && \
printf "#!/bin/bash\nsource env/bin/activate && cd ahub/src && \
python3 index.py" > /bin/init.sh && chmod +x /bin/init.sh
# expose port
EXPOSE 5002
ENTRYPOINT ["init.sh"]
================================================
FILE: aquila/encoder/LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: aquila/encoder/README.md
================================================
<div align="center">
<a href="https://aquila.network">
<img
src="https://user-images.githubusercontent.com/19545678/133918727-5a37c6be-676f-427b-8c86-dd50f58d1287.png"
alt="Aquila Network Logo"
height="64"
/>
</a>
<br />
<p>
<h3>
<b>
Aquila Hub
</b>
</h3>
</p>
<p>
<b>
Load and serve Neural Encoder Models
</b>
</p>
<br/>
</div>
Load and serve ML models to compress data into latent vectors. To be used with Aquila DB.
# Technology
Aquila Hub automates the process of encoding information with the help of ML models. Here is where Aquila Hub fits in the entire ecosystem:
<div align="center">
<img
src="https://user-images.githubusercontent.com/19545678/133918439-e08f314b-ad15-441e-a605-2fd2ec37a509.png"
alt="Aquila Hub Architecture"
height="400"
/>
<br/>
</div>
# Install
### Debian
Run `curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaHub/main/install.sh | /bin/bash -s -- -d 1 `.
### Docker
**You need docker installed in your system**
Build image (one time process): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaHub/main/Dockerfile -t aquilahub:local`
Run image (to deploy Aquila DB): `docker run -p 5002:5002 -d aquilahub:local`
================================================
FILE: aquila/encoder/authentication.py
================================================
from utils import cryptops
import os
pub_key = cryptops.read_public_key(os.environ["AUTH_KEY_FILE"])
def check (json_data, signature):
return cryptops.verify_signature(json_data, pub_key, signature)
================================================
FILE: aquila/encoder/config.yml
================================================
auth:
pubkey: "/ossl/public.pem"
ipfs:
gateway: "http://127.0.0.1:8080"
================================================
FILE: aquila/encoder/encoder.py
================================================
import logging
import fasttext
from utils import downloader
import hashlib
import base58
import json
from sentence_transformers import SentenceTransformer
import os
# define constants
MODEL_FASTTEXT = "ftxt"
MODEL_S_TRANSFORMER = "strn"
PREDICT_BATCH_SIZE = 1000
# Maintain a model directory
data_dir = os.environ["DATA_STORE_LOCATION"]
model_dir = data_dir + "models/"
model_dict = {}
def get_url (schema):
"""
Get model url from a schema
"""
if schema.get("encoder") != None:
return schema["encoder"]
else:
return None
def get_url_hash (url):
hash_ = hashlib.sha256(url.encode('utf-8'))
b58c_ = base58.b58encode(hash_.digest())
return b58c_.decode('utf-8')
def download_model (url, directory, file_name):
"""
Download a model from a URL
"""
# handle fasttext models from url or IPFS
if url.split(":")[0] == MODEL_FASTTEXT:
url = ":".join(url.split(":")[1:])
if url.split(":")[0] == "http" or url.split(":")[0] == "https":
return MODEL_FASTTEXT, downloader.http_download(url, directory, file_name+".bin")
elif url.split(":")[0] == "ipfs":
return MODEL_FASTTEXT, downloader.ipfs_download(url, directory, file_name+".bin")
elif url.split(":")[0] == MODEL_S_TRANSFORMER:
url = ":".join(url.split(":")[1:])
return MODEL_S_TRANSFORMER, url
else:
logging.error("Invalid encoder specified in schema definition.")
return None, ""
def memload_model (model_type, model_filename):
"""
Load a model from disk
"""
if model_type == MODEL_FASTTEXT:
if model_filename:
logging.debug("loading fasttext model into memory..")
return model_type, fasttext.load_model(model_filename)
else:
return None, None
elif model_type == MODEL_S_TRANSFORMER:
if model_filename:
logging.debug("loading STransformer model into memory..")
return model_type, SentenceTransformer(model_filename)
else:
return None, None
else:
return None, None
class EncodeRequest ():
def __init__(self, id_in, text_in):
self.id = id_in
self.text = text_in
class Encoder ():
def __init__(self, encoder_name_in, request_queue_in):
self.encoder_name = get_url_hash(encoder_name_in)
# to handle requests
self.request_queue = request_queue_in
self.request_id_counter = 0
self.request_id_counter_max = 10000
# to handle responses
self.response_queue = [None] * self.request_id_counter_max
def __del__(self):
logging.debug("killed encoder for database")
def count_request_id (self):
ret_ = self.request_id_counter
self.request_id_counter = (self.request_id_counter + 1) % self.request_id_counter_max
return ret_
def preload_model (self, json_schema, database_name):
"""
Download a model and load it into memory
"""
# prefill model & hash dictionary
global model_dict
try:
# load model if not done already
if not model_dict.get(self.encoder_name):
model_type_, model_file_loc_ = download_model(get_url(json_schema), model_dir, self.encoder_name)
# download success
if model_file_loc_ != None:
model_dict[self.encoder_name] = {}
# load into memory
model_dict[self.encoder_name]["type"], model_dict[self.encoder_name]["model"] = memload_model(model_type_, model_file_loc_)
# memory loading failed
if model_dict[self.encoder_name]["type"] == None:
logging.error("Memory loading of model failed")
return False
else:
return False
if model_dict[self.encoder_name].get("model"):
logging.debug("Model loaded for database: "+database_name)
return True
else:
logging.error("Model loading failed for database: "+database_name)
# reset DB - hash map
del model_dict[self.encoder_name]
return False
else:
return True
except Exception as e:
logging.error(e)
return False
async def enqueue_compress_data (self, texts):
"""
Add to request queue for compression
"""
request_ = EncodeRequest(self.count_request_id(), texts)
await self.request_queue.put(request_)
return request_.id
async def process_queue (self):
"""
Load an already existing model, pop request queue,
compress information, push to response queue
"""
request_data = []
request_metadata = []
max_batch_len = PREDICT_BATCH_SIZE # model's batching capacity
# create batch from req. queue
while(not self.request_queue.empty()):
# get an item from queue
section_ = await self.request_queue.get()
request_data += section_.text
request_metadata.append( (section_.id, len(section_.text)) )
# check max. batch length achieved
if len(request_data) > max_batch_len:
break
# prefill model & hash dictionary
global model_dict
# model_dict[self.encoder_name]
if not model_dict.get(self.encoder_name):
# try dynamic loading of model
try:
model_dict[self.encoder_name] = {}
model_dict[self.encoder_name]["type"], model_dict[self.encoder_name]["model"] = memload_model(MODEL_FASTTEXT, model_dir + self.encoder_name + ".bin")
except Exception as e:
logging.error("Model not pre-loaded for database.")
logging.error(e)
return []
result = []
try:
# fasttext model prediction
if model_dict[self.encoder_name]["type"] == MODEL_FASTTEXT:
result = []
# fasttext doesn't take in batch; so, loop it.
for line_ in request_data:
result.append(model_dict[self.encoder_name]["model"].get_sentence_vector(line_).tolist())
# stransformer model prediction
if model_dict[self.encoder_name]["type"] == MODEL_S_TRANSFORMER:
result = model_dict[self.encoder_name]["model"].encode(request_data).tolist()
except Exception as e:
logging.error(e)
logging.error("Model prediction error for database.")
# add results to response queue
self.response_queue[request_metadata[0][0]] = result[0:request_metadata[0][1]]
old_metadata = request_metadata[0]
for metadata_ in request_metadata[1:]:
self.response_queue[metadata_[0]] = result[old_metadata[1]:old_metadata[1]+metadata_[1]]
old_metadata = metadata_
================================================
FILE: aquila/encoder/index.py
================================================
import logging
from quart import Quart
from quart import request
from functools import wraps
import asyncio
from utils import config
import authentication
import manager as man_
app = Quart(__name__, instance_relative_config=True)
# Server starter
def quartserver ():
"""
start server
"""
app.run(host='0.0.0.0', port=5002, debug=False)
# Add authentication
def authenticate ():
def decorator (f):
@wraps(f)
async def wrapper (*args, **kwargs):
params = await extract_request_params(request)
if not params or not "data" in params or not "signature" in params:
return "Unauthorised access", 401
if not authentication.check(params["data"], params["signature"]):
return "Unauthorised access", 401
return await f(*args, **kwargs)
return wrapper
return decorator
async def extract_request_params (request):
if not request.is_json:
logging.error("Cannot parse request parameters")
# request is invalid
return {}
# Extract JSON data
data_ = await request.get_json()
return data_
@app.route("/", methods=['GET'])
def info ():
"""
Check server status
"""
# Build response
return {
"success": True,
"message": "AquilaHub is running healthy"
}, 200
@app.route("/prepare", methods=['POST'])
@authenticate()
async def prepare_model ():
"""
Preload and prepare model from schema definition
"""
# get parameters
params = (await extract_request_params(request)).get("data")
if not params:
# Build error response
return {
"success": False,
"message": "Invalid parameters"
}, 400
if "schema" in params:
database_name = app.manager.preload_model(params.get("schema"))
# Build response
if database_name:
return {
"success": True,
"databaseName": database_name
}, 200
else:
return {
"success": False,
"message": "Invalid schema definition"
}, 400
else:
return {
"success": False,
"message": "Invalid parameters"
}, 400
@app.route("/compress", methods=['POST'])
async def compress_data ():
"""
generate embeddings for an input data
"""
# get parameters
params = (await extract_request_params(request)).get("data")
if not params:
# Build error response
return {
"success": False,
"message": "Invalid parameters"
}, 400
if "text" in params and "databaseName" in params:
vectors = await app.manager.compress_data(params.get("databaseName"), params.get("text"))
# Build response
if vectors:
return {
"success": True,
"vectors": vectors
}, 200
else:
return {
"success": False,
"message": "Database not found"
}, 400
else:
return {
"success": False,
"message": "Invalid parameters"
}, 400
@app.before_serving
async def init_variables():
app.manager = man_.Manager()
# prepare HUB from backup
try:
app.manager.prepare_hub()
except Exception as e:
logging.error("Backup restore failed")
logging.error(e)
# initialize background task controller
app.manager.bg_task_active = True
@app.before_serving
async def init_tasks():
# initialize background task
app.manager.background_task = asyncio.ensure_future(app.manager.background_task())
# app.add_background_task(background_task)
@app.after_serving
async def shutdown():
# shutdown background task
app.manager.bg_task_active = False
app.manager.background_task.cancel()
if __name__ == "__main__":
quartserver()
================================================
FILE: aquila/encoder/ipfs.service
================================================
[Unit]
Description=IPFS daemon
After=network.target
[Service]
### custom ipfs datastore location
# Environment=IPFS_PATH=/path/to/your/ipfs/datastore
ExecStart=/usr/local/bin/ipfs daemon
Restart=on-failure
[Install]
WantedBy=default.target
================================================
FILE: aquila/encoder/manager.py
================================================
import logging
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
import asyncio
import os
import json
import copy
from utils import CID, schema
import encoder as enc_
SLEEP_PROTECTION = 0.0001
def get_database_name (schema_in):
"""
Get databse name from schema
"""
database_name = None
try:
schema_def = schema.generate_schema(schema_in)
database_name = CID.doc2CID(schema_def)
except Exception as e:
logging.error(e)
return database_name
def get_encoder_name (schema_in):
"""
Get encoder name from schema
"""
encoder_name = None
try:
schema_def = schema.generate_schema(schema_in)
encoder_name = schema_def["encoder"]
except Exception as e:
logging.error(e)
return encoder_name
class Manager ():
def __init__ (self):
# to track all database - encoder mappings
self.db_to_encoders_map = {}
self.encoders_to_obj_map = {}
def __del__ (self):
logging.debug("Killed manager")
def persist_db (self, json_schema, database_name, persist=True):
# do nothing
if not persist:
logging.debug("Skipping schema persist to Disk")
return database_name
# create backup directory
location = os.environ["DATA_STORE_LOCATION"]+"hub_backup/"
try:
if not os.path.exists(location):
# create backup directory
os.mkdir(location)
except Exception as e:
logging.error("Backup directory creation failed")
logging.error(e)
return None
# store schema, replacing the old one
try:
with open(location+database_name, "w") as f_:
json.dump(json_schema, f_)
except Exception as e:
logging.error("Schema JSON writing failed")
logging.error(e)
return None
return database_name
# initial preperation of Aquila HUB from backup
def prepare_hub (self):
# load schema files
location = os.environ["DATA_STORE_LOCATION"]+"hub_backup/"
for database_name in os.listdir(location):
with open(location+database_name, "r") as schema__:
schema_ = json.load(schema__)
database_name_ = self.preload_model(schema_, persist=False)
if database_name_ == None:
logging.debug("Model preloading failed for database: "+database_name)
elif database_name_ == database_name:
logging.debug("Model preloaded for database: "+database_name)
else:
logging.debug("Model missmatch for database "+database_name+", schema backup is modified. Hope you know what you're doing.")
def preload_model (self, json_schema, persist=True):
"""
Download a model and load it into memory
"""
# create a schema copy
schema_copy = copy.deepcopy(json_schema)
# parse schema
# NOTE: get_database_name makes modifications to "json_schema" variable
# so, beware of it's subsequent usage. If you want to use the original schema,
# use "schema_copy" variable instead
database_name = get_database_name(json_schema)
encoder_name = get_encoder_name(json_schema)
# load model for database
if database_name:
# database already not created?
if not self.db_to_encoders_map.get(database_name):
self.db_to_encoders_map[database_name] = encoder_name
# encoder already not created?
if not self.encoders_to_obj_map.get(encoder_name):
self.encoders_to_obj_map[encoder_name] = enc_.Encoder(encoder_name, asyncio.Queue())
# encoder already loaded?
if self.encoders_to_obj_map[encoder_name].preload_model(json_schema, database_name):
return self.persist_db(schema_copy, database_name, persist)
else:
# reset all, don't create DB & encoder
self.encoders_to_obj_map[encoder_name] = None
self.db_to_encoders_map[database_name] = None
return None
else:
return self.persist_db(schema_copy, database_name, persist)
else:
return self.persist_db(schema_copy, database_name, persist)
else:
return None
async def compress_data (self, database_name, texts):
"""
Load an already existing database
"""
if self.db_to_encoders_map.get(database_name):
encoder_name = self.db_to_encoders_map[database_name]
if self.encoders_to_obj_map.get(encoder_name):
response_ = None
# add compression request to queue, get request id
req_id = await self.encoders_to_obj_map[encoder_name].enqueue_compress_data(texts)
# wait until request id is processed
while (True):
# response available yet?
response_ = self.encoders_to_obj_map[encoder_name].response_queue[req_id]
if response_ != None:
# response available; take it, reset queue & break waiting
self.encoders_to_obj_map[encoder_name].response_queue[req_id] = None
break
# sleep for a while
await asyncio.sleep(SLEEP_PROTECTION)
return response_
else:
return None
else:
return None
# define background task to process request queue for each database object
async def background_task(self):
logging.debug("===== Background task INIT =====")
while self.bg_task_active:
# for each database object
for key_ in self.encoders_to_obj_map:
# any request available in queue?
if self.encoders_to_obj_map[key_].request_queue.empty():
continue
# process request
await self.encoders_to_obj_map[key_].process_queue()
await asyncio.sleep(SLEEP_PROTECTION)
================================================
FILE: aquila/encoder/requirements.txt
================================================
aiofiles==0.8.0
base58==2.1.1
blinker==1.4
bson==0.5.10
certifi==2021.10.8
chardet==4.0.0
charset-normalizer==2.0.10
click==8.0.3
fastjsonschema==2.15.3
fasttext==0.9.2
filelock==3.4.2
h11==0.12.0
h2==4.1.0
hpack==4.0.0
huggingface-hub==0.4.0
hypercorn==0.13.2
hyperframe==6.0.1
idna==3.3
itsdangerous==2.0.1
Jinja2==3.0.3
joblib==1.1.0
MarkupSafe==2.0.1
nltk==3.6.7
numpy==1.22.0
packaging==21.3
Pillow==9.0.0
priority==2.0.0
pybind11==2.9.0
pycryptodome==3.12.0
pyparsing==3.0.6
python-dateutil==2.8.2
PyYAML==6.0
quart==0.16.2
regex==2021.11.10
requests==2.27.1
sacremoses==0.0.47
scikit-learn==1.0.2
scipy==1.7.3
sentence-transformers==2.1.0
sentencepiece==0.1.96
six==1.16.0
threadpoolctl==3.0.0
tokenizers==0.10.3
toml==0.10.2
torch==1.10.1
torchvision==0.11.2
tqdm==4.62.3
transformers==4.15.0
typing-extensions==4.0.1
urllib3==1.26.8
Werkzeug==2.0.2
wsproto==1.0.0
================================================
FILE: aquila/encoder/run_tests.sh
================================================
# rm -r /data/*
python3 -m unittest test.apis.hub_fns -v
================================================
FILE: aquila/encoder/test/__init__.py
================================================
================================================
FILE: aquila/encoder/test/apis/hub_fns.py
================================================
import unittest
from utils import CID, schema
from Crypto.Hash import SHA384
from Crypto.PublicKey import RSA
from Crypto.Signature import pkcs1_15
import base58
import requests
from requests.structures import CaseInsensitiveDict
import json
import bson
host = "127.0.0.1:5002"
# load private key
with open("private_unencrypted.pem", "r") as pkf:
k = pkf.read()
priv_key = RSA.import_key(k)
class TestAuth (unittest.TestCase):
# Preload a model
def test_1_auth_preload_http (self):
schema_def_ = {
"description": "this is my database",
"unique": "r8and0mseEd905",
"encoder": "ftxt:https://ftxt-models.s3.us-east-2.amazonaws.com/ftxt_base_min.bin",
"codelen": 25,
"metadata": {}
}
# generate schema
schema_def = schema.generate_schema(schema_def_)
database_name = CID.doc2CID(schema_def)
# 1. test preperation
data_ = { "schema": schema_def_ }
data_bson = bson.dumps(data_)
# generate hash
hash = SHA384.new()
hash.update(data_bson)
# Sign with pvt key
signer = pkcs1_15.new(priv_key)
signature = signer.sign(hash)
signature = base58.b58encode(signature).decode("utf-8")
url = "http://"+host+"/prepare"
headers = CaseInsensitiveDict()
headers["Content-Type"] = "application/json"
data = {
"data": data_,
"signature": signature
}
data = json.dumps(data)
resp = requests.post(url, headers=headers, data=data)
database_name_ = resp.json()["databaseName"]
# check databases are the same
self.assertEqual(database_name, database_name_, "DB name doesn't match")
# 2. test compression
data_ = {"databaseName": database_name_, "text": ["data one", "data two"]}
url = "http://"+host+"/compress"
headers = CaseInsensitiveDict()
headers["Content-Type"] = "application/json"
data = {
"data": data_
}
data = json.dumps(data)
resp = requests.post(url, headers=headers, data=data)
# # check returned array is valid
self.assertEqual(len(resp.json()["vectors"]), len(data_["text"]), "Compressed items doesn't match query")
self.assertEqual(len(resp.json()["vectors"][0]), schema_def_["codelen"], "Compressed code length doesn't match")
# Preload a model
def test_2_auth_preload_ipfs (self):
schema_def_ = {
"description": "this is my database",
"unique": "r8and0mseEd905",
"encoder": "ftxt:ipfs://QmT9CECrTwUhAPw6VHgxcLciH4EBepkXJmSB9y2rchsVQz",
"codelen": 25,
"metadata": {}
}
# generate schema
schema_def = schema.generate_schema(schema_def_)
database_name = CID.doc2CID(schema_def)
# 1. test preperation
data_ = { "schema": schema_def_ }
data_bson = bson.dumps(data_)
# generate hash
hash = SHA384.new()
hash.update(data_bson)
# Sign with pvt key
signer = pkcs1_15.new(priv_key)
signature = signer.sign(hash)
signature = base58.b58encode(signature).decode("utf-8")
url = "http://"+host+"/prepare"
headers = CaseInsensitiveDict()
headers["Content-Type"] = "application/json"
data = {
"data": data_,
"signature": signature
}
data = json.dumps(data)
resp = requests.post(url, headers=headers, data=data)
database_name_ = resp.json()["databaseName"]
# check databases are the same
self.assertEqual(database_name, database_name_, "DB name doesn't match")
# 2. test compression
data_ = {"databaseName": data_["databaseName"], "text": ["data one", "data two"]}
url = "http://"+host+"/compress"
headers = CaseInsensitiveDict()
headers["Content-Type"] = "application/json"
data = {
"data": data_
}
data = json.dumps(data)
resp = requests.post(url, headers=headers, data=data)
# check returned array is valid
self.assertEqual(len(resp.json()["vectors"]), len(data_["text"]), "Compressed items doesn't match query")
self.assertEqual(len(resp.json()["vectors"][0]), schema_def_["codelen"], "Compressed code length doesn't match")
if __name__ == '__main__':
unittest.main()
================================================
FILE: aquila/encoder/test.py
================================================
from aquilapy import Wallet, DB, Hub
import numpy as np
import time
import multiprocessing as mp
# Create a wallet instance from private key
wallet = Wallet("/home/iamjbn/aquilax/ossl/private_unencrypted.pem")
host = "http://127.0.0.1"
# Connect to Aquila Hub instance
hub = Hub(host, "5002", wallet)
# Schema definition to be used
schema_def = {
"description": "AquilaX-CE default user index",
"unique": "user_id1",
"encoder": "ftxt:http://0.0.0.0:2000/cc.en.300.bin",
"codelen": 300,
"metadata": {
"url": "string",
"text": "string"
}
}
def run_test (index_in):
print('index_in:', index_in)
# Craete a database with the schema definition provided
db_name_ = hub.create_database(schema_def)
# Generate encodings
texts = ["Sure, technically these are processes, and this program should really be called a process spawning manager, but this is only due to the way that BASH works when it forks using the ampersand, it uses the fork() or perhaps clone() system call which clones into a separate memory space, rather than something like pthread_create() which would share memory. If BASH supported the latter, each sequence of execution would operate just the same and could be termed to be traditional threads whilst gaining a more efficient memory footprint.",
"Sure, technically these are processes, and this program should really be called a process spawning manager, but this is only due to the way that BASH works when it forks using the ampersand, it uses the fork() or perhaps clone() system call which clones into a separate memory space, rather than something like pthread_create() which would share memory. If BASH supported the latter, each sequence of execution would operate just the same and could be termed to be traditional threads whilst gaining a more efficient memory footprint.",
"Sure, technically these are processes, and this program should really be called a process spawning manager, but this is only due to the way that BASH works when it forks using the ampersand, it uses the fork() or perhaps clone() system call which clones into a separate memory space, rather than something like pthread_create() which would share memory. If BASH supported the latter, each sequence of execution would operate just the same and could be termed to be traditional threads whilst gaining a more efficient memory footprint.",
"Sure, technically these are processes, and this program should really be called a process spawning manager, but this is only due to the way that BASH works when it forks using the ampersand, it uses the fork() or perhaps clone() system call which clones into a separate memory space, rather than something like pthread_create() which would share memory. If BASH supported the latter, each sequence of execution would operate just the same and could be termed to be traditional threads whilst gaining a more efficient memory footprint."]
compression = hub.compress_documents(db_name_, texts)
print(len(compression)==len(texts))
# test concurrency
def test_concurrency_thread (counter_upto):
start = time.time()
print("starting thread")
pool = mp.Pool(500) #mp.cpu_count())
pool.map(run_test, range(counter_upto))
pool.close()
pool.join()
end = time.time()
print(end-start, counter_upto)
start = end
test_concurrency_thread(10000)
# test model downloads from different sources
schema_def["encoder"] = "ftxt:http://0.0.0.0:2000/cc.en.300.bin"
schema_def["codelen"] = 300
print(schema_def)
db_name_ = hub.create_database(schema_def)
print(db_name_)
# test model downloads from different sources
schema_def["encoder"] = "ftxt:ipfs://QmY2FFRuW4xVeCDkwgwkWcq1aHaKFjfbEHPXjEEuQYax4P"
schema_def["codelen"] = 300
db_name_ = hub.create_database(schema_def)
print(db_name_)
================================================
FILE: aquila/encoder/utils/CID.py
================================================
import logging
import bson
import hashlib
import base58
def doc2CID (inp):
try:
# JSON OBJ to BSON encode
bson_ = bson.dumps(inp)
# SHA-256 Double Hashing
hash_ = hashlib.sha256(bson_)
hash_ = hashlib.sha256(hash_.digest())
# Convert to Base-58 string
b58c_ = base58.b58encode(hash_.digest())
return b58c_.decode('utf-8')
except Exception as e:
logging.debug(e)
return None
def doc2bson (inp):
try:
# JSON OBJ to BSON encode
bson_ = bson.dumps(inp)
return bson_
except Exception as e:
logging.debug(e)
return None
def bson2doc (inp):
try:
# BSON to JSON OBJ
json_ = bson.loads(inp)
return json_
except Exception as e:
logging.debug(e)
return None
================================================
FILE: aquila/encoder/utils/config.py
================================================
import yaml
import os
os.environ["DATA_STORE_LOCATION"] = "/data/"
with open("config.yml", "r") as stream:
DB_config = yaml.safe_load(stream)
if "AUTH_KEY_FILE" not in os.environ:
os.environ["AUTH_KEY_FILE"] = str(DB_config["auth"]["pubkey"])
if "IPFS_GATEWAY" not in os.environ:
os.environ["IPFS_GATEWAY"] = str(DB_config["ipfs"]["gateway"])
================================================
FILE: aquila/encoder/utils/cryptops.py
================================================
import logging
from Crypto.Hash import SHA384
from Crypto.PublicKey import RSA
from Crypto.Signature import pkcs1_15
import base58
import chardet
import bson
def read_public_key (location):
# disk read public key
with open(location, "r") as pkf:
k = pkf.read()
pub_key = RSA.import_key(k)
return pub_key
def verify_signature (json_data, pub_key, signature):
ret = True
binary_data = bson.dumps(json_data)
# generate hash
hash = SHA384.new()
hash.update(binary_data)
signature = base58.b58decode(signature)
# Verify with pub key
verifier = pkcs1_15.new(pub_key)
try:
verifier.verify(hash, signature)
except Exception as e:
logging.debug(e)
ret = False
return ret
================================================
FILE: aquila/encoder/utils/downloader.py
================================================
import logging
import os
import time
import requests
from tqdm import tqdm
def download_large_file (url, location, method="get"):
r = None
if method == "get":
r = requests.get(url, stream=True)
if method == "post":
r = requests.post(url, stream=True)
if r != None:
r.raise_for_status()
total_size_in_bytes= int(r.headers.get('content-length', 0))
block_size = 1024*5
progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
with open(location, 'wb') as f:
for chunk in r.iter_content(chunk_size=block_size):
progress_bar.update(len(chunk))
f.write(chunk)
r.close()
progress_bar.close()
if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
logging.error("Download failed.")
os.remove(location)
return False
return True
def http_download (url, directory, file_name):
"""
Download model via http
"""
try:
# create models dir, if not exists
if not os.path.exists(directory):
os.makedirs(directory)
original_bin_file_name = url.split("/")[-1]
# check if model already exists
if not os.path.exists(directory+original_bin_file_name):
# download file
logging.debug("Downloading model..")
# download from given url
status = download_large_file(url, directory+original_bin_file_name)
# failed??
if not status:
return None
if not os.path.exists(directory+file_name):
# copy and rename model
logging.debug("Copy model..")
os.symlink(directory+original_bin_file_name, directory+file_name)
except Exception as e:
logging.error(e)
return None
return directory+file_name
def ipfs_download (url, directory, file_name):
"""
Download model via IPFS
"""
try:
# create models dir, if not exists
if not os.path.exists(directory):
os.makedirs(directory)
IPFS_CID = url.split("ipfs://")[1]
original_bin_file_name = IPFS_CID
# check if model already exists
if not os.path.exists(directory+original_bin_file_name):
# download file
logging.debug("Downloading model..")
logging.debug("Connecting to local IPFS demon..")
# download from IPFS local API
url_ = os.environ["IPFS_GATEWAY"]+"/ipfs/"+IPFS_CID
status = download_large_file(url_, directory+original_bin_file_name)
# failed??
if not status:
return None
if not os.path.exists(directory+file_name):
# copy and rename model
logging.debug("Copy model..")
os.symlink(directory+original_bin_file_name, directory+file_name)
except Exception as e:
logging.error(e)
return None
return directory+file_name
================================================
FILE: aquila/encoder/utils/schema.py
================================================
import logging
import fastjsonschema
def generate_schema (template):
# get all keys from template schema
metadata_templ = template.get("metadata")
if metadata_templ:
del template["metadata"]
else:
metadata_templ = {}
keys_ = list(template.keys())
# sort keys
keys_.sort()
# validate required keys
req_k = ["description", "encoder", "unique"]
for r_ in req_k:
if r_ not in keys_:
# cannot continue, invalid schema
return None
# generate schema in sorted keys
generated_schema = {
"$schema": "http://json-schema.org/schema#",
"$id": "http://aquilanetwork.com/schema/id/v1",
"title": "Schema",
"description": "",
"encoder": "",
"unique": "",
"type": "object",
"properties": {
"code": {
"description": "Encoded data",
"type": "array",
"items": {
"type": "number"
}
},
"metadata": {
"$ref": "#/definitions/metadata"
}
},
"definitions": {
"metadata": {
"description": "User defined metadata",
"type": "object",
"properties": {},
"required": []
}
},
"required": ["code", "metadata"]
}
metadata = {}
metadata_types = ["number", "string"]
for key_ in keys_:
if key_ in req_k:
# fill in root level keys
generated_schema[key_] = template[key_]
elif key_ == "codelen":
generated_schema["properties"]["code"]["maxItems"] = template[key_]
else:
# fill in root level keys :: extra info that we don't care
generated_schema[key_] = template[key_]
for key_ in metadata_templ.keys():
# check type is predefined
if metadata_templ[key_] not in metadata_types:
return None
# fill in metadata keys
metadata[key_] = {
"type": metadata_templ[key_]
}
generated_schema["definitions"]["metadata"]["properties"] = metadata
generated_schema["definitions"]["metadata"]["required"] = list(metadata.keys())
# validate values
if type(generated_schema["properties"]["code"].get("maxItems")) != int:
return None
return generated_schema
def compile (schema_def):
# compile schema
validator = fastjsonschema.compile(schema_def)
return validator
def validate_json_docs (validator, json_doc):
try:
# validate doc on schema :: will except on fail
validator(json_doc)
# validated
# logging.debug("Schema validation success")
return True
except Exception as e:
logging.error(e)
return False
================================================
FILE: aquila/metricstore/.travis.yml
================================================
language: node_js
services:
- docker
node_js:
- "11"
before_install:
- docker build -f Dockerfile_local_build -t ammaorg/aquiladb:travis .
- docker run -d -i -p 50051:50051 -t ammaorg/aquiladb:travis
- docker ps -a
- sudo apt-get install -y make
- cd src
install:
- npm install
- cd test
script:
- node test.js
branches:
only:
- develop
================================================
FILE: aquila/metricstore/CODE_OF_CONDUCT.md
================================================
# AquilaDB Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, sex characteristics, gender identity and expression,
level of experience, education, socio-economic status, nationality, personal
appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at humans@aquiladb.xyz. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq
================================================
FILE: aquila/metricstore/DB_config.yml
================================================
docs:
cswap: 10101 # minimum data required to start indexing
vd: 784 # fixed vector dimension
faiss:
init:
nlist: 1 # number of cells
nprobe: 1 # number of cells that are visited to perform a search
bpv: 8 # bytes per vector
bpsv: 8 # bytes per sub vector
annoy:
init:
smetric: "dot" # similarity metric to be used
ntrees: 500 # no. of trees
search_k: -1 # search_k is provided in runtime and affects the search performance. A larger value will give more accurate results, but will take longer time to return.
queue:
qlen: 100000 # length limit for quesues used
sleep: 1 # thread waiting in seconds
auth:
pubkey: "/ossl/public.pem"
================================================
FILE: aquila/metricstore/Dockerfile
================================================
# start a new build stage
FROM ubuntu:latest as builder
RUN ls
# set work directory
ENV ROOT_DIR /home/root
WORKDIR $ROOT_DIR
# install aquiladb
RUN apt update && apt install -y curl && \
curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/install.sh | /bin/bash
# preperations
ENV PATH="$ROOT_DIR/env/bin:$PATH"
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
# install and start demon
RUN mkdir -p /data && \
printf "#!/bin/bash\nsource env/bin/activate && export MINI_AQDB='active' && cd adb/src && \
python3 index.py" > /bin/init.sh && chmod +x /bin/init.sh
# expose port
EXPOSE 5001
ENTRYPOINT ["init.sh"]
================================================
FILE: aquila/metricstore/DockerfileBig
================================================
# start a new build stage
FROM ubuntu:latest as builder
# set work directory
ENV ROOT_DIR /home/root
WORKDIR $ROOT_DIR
# install aquiladb
RUN apt update && apt install -y curl && \
curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/install.sh | /bin/bash -s -- -m 0
# start a new runner stage
FROM ubuntu:latest as runner
# set work directory
ENV ROOT_DIR /home/root
WORKDIR $ROOT_DIR
RUN echo "$ROOT_DIR"
# copy required files from builder stage
COPY --from=builder $ROOT_DIR/env $ROOT_DIR/env
COPY --from=builder $ROOT_DIR/adb $ROOT_DIR/adb
COPY --from=builder /ossl /ossl
# preperations
ENV PATH="$ROOT_DIR/env/bin:$PATH"
WORKDIR $ROOT_DIR
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
# install and start demon
RUN export DEBIAN_FRONTEND=noninteractive && mkdir -p /data && apt update && \
apt install -y python3 libgomp1 libblas-dev liblapack-dev && \
printf "#!/bin/bash\nsource env/bin/activate && cd adb/src && \
python3 index.py" > /bin/init.sh && chmod +x /bin/init.sh
# expose port
EXPOSE 5001
ENTRYPOINT ["init.sh"]
================================================
FILE: aquila/metricstore/LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: aquila/metricstore/README.md
================================================
<div align="center">
<a href="https://aquila.network">
<img
src="https://user-images.githubusercontent.com/19545678/133918727-5a37c6be-676f-427b-8c86-dd50f58d1287.png"
alt="Aquila Network Logo"
height="64"
/>
</a>
<br />
<p>
<h3>
<b>
Aquila DB
</b>
</h3>
</p>
<p>
<b>
Easy to use Neural Search Engine
</b>
</p>
<br/>
</div>
**[Aquila DB](https://github.com/Aquila-Network/AquilaDB)** is a Neural search engine. In other words, it is a database to index **Latent Vectors** generated by ML models along with **JSON Metadata** to perform **k-NN** retrieval. It is dead simple to set up, language-agnostic, and drop in addition to your Machine Learning Applications. Aquila DB, as of current features is a ready solution for Machine Learning engineers and Data scientists to build **[Neural Information Retrieval](https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf)** applications out of the box with minimal dependencies.
> This project is still in alpha version & we're already using it in production to power semantic search at https://aquila.network.
Wanna support this project? Yes, we love getting a **star** ⭐ and **shout-out** 🗣️ 🤗
Join [Community chat and get support: ](https://discord.gg/5YP7zHS)
# Who is this for
* If you are working on a data science project and need to store a hell of a lot of data and retrieve similar data based on some feature vector, this will be a useful tool to you, with extra benefits a real world web application needs.
* Are you dealing with a lot of images and related metadata? Want to find similar ones? You are at the right place.
* If you are looking for a document database, this is not the right place for you.
# Technology
Aquila DB powers search features of Aquila Network. Here is where Aquila DB fits in the entire ecosystem:
<div align="center">
<img
src="https://user-images.githubusercontent.com/19545678/133918438-997af8ec-67ef-4f7c-95ab-16d2f28e79e3.png"
alt="Aquila DB Architecture"
height="400"
/>
<br/>
</div>
If you are serious and wanna dive down the rabbit hole, read our **[whitepapers](https://github.com/Aquila-Network/whitepaper)** and **[technical specifications](https://github.com/Aquila-Network/specs)** (being actively worked on).
**As a side note**, everything in **[Aquila Network](https://github.com/Aquila-Network)** is defined by the specifications and a large chunk of our efforts goes into it. We also maintain quality implementations of those specifications with non-technical users in mind. This is to make sure that - Aquila Network is fully open, decentralized by design, and Fair. You can follow those specifications to implement your alternative software and still interact with the network without any restrictions.
# Install
### Debian
Run `curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/install.sh | /bin/bash -s -- -d 1 `.
### Docker
**You need docker installed in your system**
Build image (lite): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/Dockerfile -t aquiladb:local`
Build image (big data): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/DockerfileBig -t aquiladb:localbg`
Run image (to deploy Aquila DB lite): `docker run -p 5001:5001 -d aquiladb:local`
Run image (to deploy Aquila DB big): `docker run -p 5001:5001 -d aquiladb:localbg`
# Client SDKs
We currently have multiple client libraries in progress to abstract the communication between deployed Aquila DB and your applications.
[Python](https://github.com/Aquila-Network/AquilaPy)
[Node JS](https://github.com/Aquila-Network/AquilaJS)
## Where to get private key (wallet key) for client authentication
When you use a client library to authenticate with AquilaDB, you might need access the same private key (wallet key) used by AquilaDB. This key is located inside `/ossl/` directory within AquilaDB docker container (in your computer if you have installed AquilaDB directly without docker). To access the keys inside your AquilaDB container, follow below steps:
* identify `CONTAINER ID` for the already running `aquiladb` docker instance:
`docker ps`
* take a copy of private keys from docker container to your host machine:
`docker cp CONTAINER_ID:/ossl/ ./`
* now you will see a new directory named `ossl` at your current location. Use the keys inside it.
#### tips for advanced users
If your pipeline requires the private keys to be generated in advance, you can do it in your host machine and then mount it to the container's `/ossl/` directory.
Run:
```
mkdir -p <host>/ossl/
openssl genrsa -passout pass:1234 -des3 -out <host>/ossl/private.pem 2048
openssl rsa -passin pass:1234 -in <host>/ossl/private.pem -outform PEM -pubout -out <host>/ossl/public.pem
openssl rsa -passin pass:1234 -in <host>/ossl/private.pem -out <host>/ossl/private_unencrypted.pem -outform PEM
```
# Progress
This project is still and will be under active development with intermediate production releases. It can either be used as a standalone database or as a participating node in Aquila Network. Please note, [Aquila Port](https://github.com/Aquila-Network/specs/blob/main/README.md#aquila-port) (peer-peer network layer for Aquila DB nodes) is also a work in progress. Currently, you need to deploy your custom models to feed vector embeddings to Aquila DB, until [Aquila Hub](https://github.com/Aquila-Network/specs/blob/main/README.md#aquila-hub) developments get started.
# Contribute
We have [prepared a document](https://docs.google.com/document/d/1bT2_9FQIxQpx_rdYbkTukn_DJRi_haVK_ixTf8uTaDE/edit?usp=sharing) to get anyone interested to contribute, immediately started with Aquila DB.
Here is our high-level [release roadmap](https://user-images.githubusercontent.com/19545678/62313851-5af82880-b4af-11e9-84f6-21e24bf46e8a.png).
# Learn
We have started meeting developers and do small talks on Aquila DB. Here are the slides that we use on those occasions: http://bit.ly/AquilaDB-slides
**Video:**
[<img alt="introduction to Neural Information retrieval with AquilaDB" src="http://img.youtube.com/vi/-VYpjpLXU5Q/0.jpg" width="300" />](http://www.youtube.com/watch?v=-VYpjpLXU5Q)
As of current AquilaDB release features, you can build **[Neural Information Retrieval](https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf)** applications out of the box without any external dependencies. Here are some useful links to learn more about it and start building:
* Microsoft published a paper and youtube video on this to onboard anyone interested:
* paper: https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf
* video: https://www.youtube.com/watch?v=g1Pgo5yTIKg
* Embeddings for Everything: Search in the Neural Network Era: https://www.youtube.com/watch?v=JGHVJXP9NHw
* Autoencoders are one such deep learning algorithms that will help you to build semantic vectors - foundation for Neural Information retrieval. Here are some links to Autoencoders based IR:
* go to chapter 15 in this link: https://www.cs.toronto.edu/~hinton/coursera_lectures.html
* https://www.coursera.org/lecture/ml-foundations/examples-of-document-retrieval-in-action-CW25H
* https://www.coursera.org/lecture/intro-to-deep-learning/autoencoders-101-QqBOa
* Note that, the idea of information retrieval applies not only to text data but for any data. All you need to do is, encode any source datatype to a dense vector with deep neural networks.
<br/><br/>
<h1 align="center">Our Sponsors</h1>
<p align="center"><b></b></p>
<br/>
> email us to sponsor this project [adbadmin@protonmail.ch](mailto:adbadmin@protonmail.ch).
<br/><br/>
# Citing Aquila DB
If you use Aquila DB in an academic paper, we would 😍 to be cited. Here are the two ways of citing Aquila DB:
```
\footnote{https://github.com/Aquila-Network/AquilaDB}
```
```
@misc{AquilaNetwork2019AquilaDB,
title={AquilaDB: Neural Search Engine},
author={Jubin Jose, Nibin Peter},
howpublished={\url{https://github.com/Aquila-Network/AquilaDB}},
year={2019}
}
```
# License
Apache License 2.0 [license file](https://github.com/Aquila-Network/AquilaDB/blob/master/LICENSE)
created by ❤️ with a-mma (a_മ്മ)
================================================
FILE: aquila/metricstore/authentication.py
================================================
from utils import cryptops
import os
pub_key = cryptops.read_public_key(os.environ["AUTH_KEY_FILE"])
def check (json_data, signature):
return cryptops.verify_signature(json_data, pub_key, signature)
================================================
FILE: aquila/metricstore/index.py
================================================
import logging
from flask import Flask, request
from flask_cors import CORS
from flask import jsonify
from functools import wraps
from utils import config
import authentication
import router
app = Flask(__name__, instance_relative_config=True)
# Enable CORS
CORS(app)
# preload databases
router.preload_databases()
# Add authentication
def authenticate ():
def decorator (f):
@wraps(f)
def wrapper (*args, **kwargs):
params = extract_request_params(request)
if not params or not "data" in params or not "signature" in params:
return "Unauthorised access", 401
if not authentication.check(params["data"], params["signature"]):
return "Unauthorised access", 401
return f(*args, **kwargs)
return wrapper
return decorator
def extract_request_params (request):
if not request.is_json:
logging.error("Cannot parse request parameters")
# request is invalid
return None
# Extract JSON data
data_ = request.get_json()
return data_
@app.route("/", methods=['GET'])
def info ():
"""
Check server status
"""
# Build response
return {
"success": True,
"message": "AquilaDB is running healthy"
}, 200
@app.route("/db/create", methods=['POST'])
@authenticate()
def db_create ():
"""
Create database from schema definition
"""
# get parameters
params = extract_request_params(request)["data"]
if not params:
# Build error response
return {
"success": False,
"message": "Invalid parameters"
}, 400
if "schema" in params:
database_name = router.create_database(params.get("schema"))
# Build response
if database_name:
return {
"success": True,
"database_name": database_name
}, 200
else:
return {
"success": False,
"message": "Invalid schema definition"
}, 400
@app.route("/db/doc/insert", methods=['POST'])
@authenticate()
def doc_insert ():
"""
insert documents
"""
# get parameters
params = extract_request_params(request)["data"]
if not params:
# Build error response
return {
"success": False,
"message": "Invalid parameters"
}, 400
if "docs" in params and "database_name" in params:
cids = router.insert_docs(params.get("docs"), params.get("database_name"))
# Build response
return {
"success": True,
"ids": cids
}, 200
@app.route("/db/doc/delete", methods=['POST'])
@authenticate()
def doc_delete ():
"""
delete documents by cid
"""
# get parameters
params = extract_request_params(request)["data"]
if not params:
# Build error response
return {
"success": False,
"message": "Invalid parameters"
}, 400
if "ids" in params and "database_name" in params:
ids = router.delete_docs(params.get("ids"), params.get("database_name"))
# Build response
return {
"success": True,
"ids": ids
}, 200
@app.route("/db/search", methods=['GET'])
@authenticate()
def db_search ():
"""
search documents
"""
# get parameters
params = extract_request_params(request)["data"]
if not params:
# Build error response
return {
"success": False,
"message": "Invalid parameters"
}, 400
if "matrix" in params and "k" in params and "database_name" in params:
docs, dists = router.search(params.get("matrix"), params.get("k"), None, params.get("database_name"))
# Build response
return {
"success": True,
"docs": docs,
"dists": dists
}, 200
def flaskserver ():
"""
start server
"""
app.run(host='0.0.0.0', port=5001, debug=False)
if __name__ == "__main__":
flaskserver()
================================================
FILE: aquila/metricstore/install.sh
================================================
#!/bin/bash -e
export DEBIAN_FRONTEND=noninteractive
apt update
export USER=$(whoami)
export ROOT_DIR=/home/$USER
mkdir -p /data/
mkdir -p $ROOT_DIR
cd $ROOT_DIR
mini=1
gpu=0
test=0
demon=0
while getopts m:g:t:d: flag
do
case "${flag}" in
m) mini=${OPTARG};;
g) gpu=${OPTARG};;
t) test=${OPTARG};;
d) demon=${OPTARG};;
esac
done
if [[ $mini -eq 1 ]]; # if minimal install enabled
then
echo "Aquila DB minimal install will be done by default. \
Minimal install is recommended for personal use - \
install process is fast and lightweight. If you are planning to deal \
with big-data, disable minimal install with '-m 0' argument"
fi
# system packs install
apt install -y git wget nano python3 python3-pip libssl-dev
if [[ $mini -eq 0 ]]; # if minimal install disabled
then
apt install -y libblas-dev liblapack-dev swig
fi
# setup venv
pip3 install virtualenv
virtualenv $ROOT_DIR/env
source $ROOT_DIR/env/bin/activate
# install python packages
pip3 install numpy pycryptodome base58 chardet Flask requests flask_cors PyYAML bson fastjsonschema annoy plyvel
if [[ $mini -eq 0 ]]; # if minimal install disabled
then
# install cmake
apt purge --auto-remove cmake
version=3.19
build=1
mkdir -p $ROOT_DIR/temp
cd $ROOT_DIR/temp
wget https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
tar -xzvf cmake-$version.$build.tar.gz
cd cmake-$version.$build/
./bootstrap
make -j$(nproc)
make install
cmake --version
# install faiss
cd $ROOT_DIR
mkdir -p faiss
cd faiss
git clone https://github.com/facebookresearch/faiss.git .
if [[ $gpu -eq 0 ]]; # if gpu not enabled
then
echo "build FAISS without GPU"
cmake -DFAISS_ENABLE_GPU=OFF -B build .
else
echo "build FAISS with GPU"
cmake -B build .
fi
make -C build
# For the Python interface:
cd build/faiss/python
python setup.py install
cp -r $ROOT_DIR/faiss/build/faiss/python/ $ROOT_DIR/env/lib/python3.8/site-packages/faiss
fi
# clone & test AquilaDB
cd $ROOT_DIR
mkdir -p adb
cd adb
git clone https://github.com/Aquila-Network/AquilaDB.git .
mkdir -p /ossl/
openssl genrsa -passout pass:1234 -des3 -out /ossl/private.pem 2048
openssl rsa -passin pass:1234 -in /ossl/private.pem -outform PEM -pubout -out /ossl/public.pem
openssl rsa -passin pass:1234 -in /ossl/private.pem -out /ossl/private_unencrypted.pem -outform PEM
cd $ROOT_DIR/adb/src
if [[ $test -eq 1 ]]; # if tests enabled
then
chmod +x run_tests.sh
./run_tests.sh
else
echo "Not running tests"
fi
echo "==================================="
echo " AquilaDB setup complete. "
echo "==================================="
if [[ $demon -eq 1 ]]; # if demon run enabled
then
# install node
apt install -y nodejs npm
# install pm2
npm i pm2 -g
# start server
pm2 start index.py
# Keep logs alive
pm2 logs -f
fi
================================================
FILE: aquila/metricstore/manager.py
================================================
import logging
from utils import CID, schema
from vec_index import hannoy
import os
is_mini_instance = os.environ["MINI_AQDB"]
if is_mini_instance == "inactive":
from vec_index import hfaiss
import plyvel
import numpy as np
import json
import random
import threading
import queue
import time
import pickle
INDEX_LABEL = ["annoy", "faiss"]
STORE_LOCATION = os.environ["DATA_STORE_LOCATION"]
TRAIN_DAT_LEN = int(os.environ["MIN_SAWP_COUNT"])
PROCESS_TIMEOUT = int(os.environ["THREAD_SLEEP"])
MAX_Q_LEN = int(os.environ["FIXED_Q_LEN"])
def byt (inp):
return bytes(str(inp), 'ascii')
class VecManager:
def __init__ (self, json_schema):
# get database name from schema CID
database_name = CID.doc2CID(json_schema)
# keep database name
self.database_name = database_name
# set DB disk location
self.DB_disk_location = STORE_LOCATION + database_name
# create data directory for database
if not os.path.exists(self.DB_disk_location):
os.makedirs(self.DB_disk_location)
# keep schema in store location
with open(self.DB_disk_location + '/schema.json', 'w') as oschema:
json.dump(json_schema, oschema)
# get vector index
self.active_index = INDEX_LABEL[0]
self.index = self.get_index(self.DB_disk_location)
# Create KV store instance
self.KV_store = plyvel.DB(self.DB_disk_location + "/kv.db", create_if_missing=True)
if self.KV_store.get(byt(-1)) == None:
self.KV_store.put(byt(-1), byt(0))
# Training data holder
self.training_data = []
self.TD_location = self.DB_disk_location + "/TD"
# Try loading training data
self.load_TD_from_disk()
# spawn worker thread
self.q_maxsize = MAX_Q_LEN
self.process_flag = True
self.process_timeout_sec = PROCESS_TIMEOUT
self.spawn()
def __del__(self):
self.process_flag = False
if self.process_thread:
self.process_thread.join()
logging.debug("Thread stopped")
# close level database connection
self.KV_store.close()
def add_vectors (self, documents):
# add to KV store
next_index = int(self.KV_store.get(byt(-1)))
# check if it is ready to swap index
if is_mini_instance == "inactive" and next_index > TRAIN_DAT_LEN \
and self.active_index == INDEX_LABEL[0] \
and len(self.training_data) >= TRAIN_DAT_LEN:
# swap index
self.swap_index(self.DB_disk_location)
# init batch write to DB
wb_ = self.KV_store.write_batch()
for idx_, doc_ in enumerate(documents):
cid_ = byt(doc_["CID"])
# resize "code"
doc_["code"] = self.resize_vector(doc_["code"], int(os.environ["FIXED_VEC_DIMENSION"]))
# cod_ = pickle.dumps(cod_)
documents[idx_]["_id"] = next_index
# TBD: convert to bulk insert
wb_.put(byt(next_index), byt(len(cid_)) + cid_ + CID.doc2bson(doc_))
wb_.put(cid_, byt(next_index))
next_index += 1
wb_.put(byt(-1), byt(next_index))
# commit DB write
wb_.write()
# push to training data
self.update_training_data(documents)
# add vectors to index
return self.index.add_vectors(documents)
def delete_vectors (self, cids):
# get ids from cids
ids_ = []
wb_ = self.KV_store.write_batch()
for cid_ in cids:
id_ = self.KV_store.get(byt(cid_))
if id_:
ids_.append(int(id_))
wb_.delete(id_)
# commit db write
wb_.write()
# delete vectors by ID from index
status, ids = self.index.delete_vectors(ids_)
if status and len(ids) == len(cids):
return cids
else:
return []
def get_nearest (self, qmatrix, k, rad):
ids = []
dists = []
qmatrix = self.resize_matrix(qmatrix, int(os.environ["FIXED_VEC_DIMENSION"]))
# radius defined,
if rad is not None:
if k is not None:
ids, dists = self.index.get_nearest_rad(qmatrix, rad)
else:
ids, dists = self.index.get_nearest_rad(qmatrix, rad)[:k]
else:
ids, dists = self.index.get_nearest_k(qmatrix, k)
# get docs
for idx_, idb in enumerate(ids):
for idx__, id_ in enumerate(idb):
value = self.KV_store.get(byt(id_))
if value:
cid_len_ = int(value[:2]) + 2
ids[idx_][idx__] = CID.bson2doc(value[cid_len_:])
else:
ids[idx_][idx__] = None
return ids, dists
def get_index (self, location):
index = None
annoy_location = location + "/h_annoy"
faiss_location = location + "/h_faiss"
if is_mini_instance == "inactive":
# try loading faiss
index = hfaiss.Faiss(faiss_location)
# check if faiss is not loaded,
if not index.is_initiated():
# destruct faiss
del index
# load annoy
index = hannoy.Annoy(annoy_location)
self.active_index = INDEX_LABEL[0]
else:
self.active_index = INDEX_LABEL[1]
else:
# load annoy
index = hannoy.Annoy(annoy_location)
self.active_index = INDEX_LABEL[0]
return index
def resize_vector (self, vector, dim):
# resize vectors
vector_l = len(vector)
# check if the vector length is below dimention limit
# then pad vector with 0 by dimension
if vector_l < dim:
vector.extend([0]*(dim-vector_l))
# make sure vector length doesn't exceed dimension limit
vector = vector[:dim]
return vector
def resize_matrix (self, matrix_, dim):
# numpize
matrix = np.array(matrix_)
# check for valid dimensions
if matrix.ndim < 2:
matrix = np.array([matrix_])
elif matrix.ndim > 2:
logging.error("Invalid query dimensions")
return [[]]
# resize vectors
vector_l = len(matrix_[0])
# check if the vector length is below dimention limit
# then pad vector with 0 by dimension
if vector_l < dim:
matrix = np.pad(matrix, ((0, 0), (0, dim-vector_l)))
# make sure vector length doesn't exceed dimension limit
matrix = matrix[:, :dim]
# listize
return matrix.tolist()
def swap_index (self, location):
logging.debug("Swapping index to FAISS")
faiss_location = location + "/h_faiss"
# init faiss index
self.index = hfaiss.Faiss(faiss_location)
# train faiss index
self.index.init_faiss(self.training_data)
# migrate data
for idx_ in range(int(self.KV_store.get(byt(-1)))):
value = self.KV_store.get(byt(idx_))
if value:
cid_len_ = int(value[:2]) + 2
self.index.add_vectors([{
"_id": int(idx_),
"code": CID.bson2doc(value[cid_len_:])["code"]
}])
# set active index
self.active_index = INDEX_LABEL[1]
def update_training_data (self, documents):
# insert new vectors to array
self.pipeline.put(documents)
def save_TD_to_disk (self):
try:
# write index
np.save(self.TD_location, np.array(self.training_data))
logging.debug('Training data writing success')
return True
except Exception as e:
logging.error('Training data writing failed' + str(e))
return False
def load_TD_from_disk (self):
try:
# load training data
self.training_data = np.load(self.TD_location+'.npy').tolist()
logging.debug('Training data loaded successfully')
return True
except Exception as e:
logging.error('Training data loading failed' + str(e))
self.training_data = []
return False
def process (self):
while (self.process_flag):
# set a timeout
time.sleep(self.process_timeout_sec)
# check if queue is not empty
if not self.pipeline.empty():
# fetch all currently available codes from queue
while not self.pipeline.empty():
# pop available codes
for doc_ in self.pipeline.get_nowait():
self.training_data.append(doc_["code"])
# shuffle array
random.shuffle(self.training_data)
# resize array
self.training_data = self.training_data[:TRAIN_DAT_LEN]
# write to disk
self.save_TD_to_disk()
def spawn (self):
# create pipeline to add documents
self.pipeline = queue.Queue(maxsize=self.q_maxsize)
# create process thread
self.process_thread = threading.Thread(target=self.process, args=(), daemon=True)
# start process thread
self.process_thread.start()
================================================
FILE: aquila/metricstore/router.py
================================================
import logging
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
import manager
import os
import json
from utils import CID, schema
STORE_LOCATION = os.environ["DATA_STORE_LOCATION"]
# databases dictionary
databases = {}
def preload_databases ():
"""
Load available databases from disk
"""
# list database names
for content_ in os.listdir(STORE_LOCATION):
# simple validation for database name (CID lenngth)
if len(content_) >= 43: # TODO: 43 check is a bad method, replace it soon
database_name = content_
with open(STORE_LOCATION + database_name + "/schema.json") as ischema:
json_schema = json.load(ischema)
manager_h = manager.VecManager(json_schema)
# load and create databases dict
validator_fn = schema.compile(json_schema)
databases[database_name] = {
"manager_h": manager_h,
"schema": {
"json": json_schema,
"validator": validator_fn
}
}
logging.debug("Loaded all existing databases")
def create_database (json_schema):
"""
Create a database from a given valid JSON schema
"""
# TBD: write ahead logging (INIT)
# generate proper schema definition from templete schema
json_schema = schema.generate_schema(json_schema)
# identify invalid schema template
if json_schema == None:
return None
# Check if database already exists
database_name = CID.doc2CID(json_schema)
if databases.get(database_name):
# return database name
logging.debug("Database already exists")
return database_name
# If database doesn't exist already,
# then create one
manager_h = manager.VecManager(json_schema)
database_name = manager_h.database_name
validator_fn = schema.compile(json_schema)
databases[database_name] = {
"manager_h": manager_h,
"schema": {
"json": json_schema,
"validator": validator_fn
}
}
# TBD: save schema to storage
# TBD: write ahead logging (END)
return database_name
def load_database (database_name):
"""
Load an already existing database
"""
return databases.get(database_name)
def insert_docs (docs, database_name):
"""
Insert a set of valid documents to database
"""
# write ahead log (INIT)
cids_ = []
docs_ = []
# get manager_h for database_name
database_h = load_database(database_name)
# invalid database name
if not database_h:
logging.debug("Database doesn't exist. Please create one.")
return cids_
# validate docs against schema
# and add CID
for doc_ in docs:
payload = doc_["payload"]
if schema.validate_json_docs(database_h["schema"]["validator"], payload):
CID_ = CID.doc2CID(payload)
cids_.append(CID_)
payload["CID"] = CID_
docs_.append(payload)
else:
cids_.append(None)
# get manager_h for database_name
manager_h = database_h["manager_h"]
manager_h.add_vectors(docs_)
# write ahead log (END)
return cids_
def delete_docs (ids, database_name):
# write ahead log (INIT)
# get manager_h for database_name
database_h = load_database(database_name)
# invalid database name
if not database_h:
logging.debug("Database doesn't exist. Please create one.")
return []
return database_h["manager_h"].delete_vectors(ids)
def search (matrix, k, rad, database_name):
# write ahead log (INIT)
# get manager_h for database_name
database_h = load_database(database_name)
return database_h["manager_h"].get_nearest(matrix, k, rad)
================================================
FILE: aquila/metricstore/run_tests.sh
================================================
rm -r /data/*
python3 -m unittest test.apis.db_fns -v
python3 -m unittest test.apis.doc_fns -v
python3 -m unittest test.apis.search_fns -v
python3 -m unittest test.apis.auth_fns -v
================================================
FILE: aquila/metricstore/test/__init__.py
================================================
================================================
FILE: aquila/metricstore/test/apis/auth_fns.py
================================================
import unittest
import index
from utils import CID, schema
from Crypto.Hash import SHA384
from Crypto.PublicKey import RSA
from Crypto.Signature import pkcs1_15
import base58
import requests
from requests.structures import CaseInsensitiveDict
import json
import bson
from multiprocessing import Process
# load private key
with open("/ossl/private_unencrypted.pem", "r") as pkf:
k = pkf.read()
priv_key = RSA.import_key(k)
class TestAuth (unittest.TestCase):
# A fresh DB is created
def test_1_auth_create_db (self):
# deploy app
server = Process(target=index.flaskserver)
server.start()
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd905",
"encoder": "example.com/autoencoder/API",
"codelen": 30,
"metadata": {
"name": "string",
"age": "number"
}
}
data_ = { "schema": schema_def }
data_bson = bson.dumps(data_)
# generate hash
hash = SHA384.new()
hash.update(data_bson)
# Sign with pvt key
signer = pkcs1_15.new(priv_key)
signature = signer.sign(hash)
signature = base58.b58encode(signature).decode("utf-8")
url = "http://127.0.0.1:5001/db/create"
headers = CaseInsensitiveDict()
headers["Content-Type"] = "application/json"
data = {
"data": data_,
"signature": signature
}
data = json.dumps(data)
resp = requests.post(url, headers=headers, data=data)
database_name_ = resp.json()["database_name"]
schema_def = schema.generate_schema(schema_def)
database_name = CID.doc2CID(schema_def)
server.terminate()
server.join()
self.assertEqual(database_name, database_name_, "DB name doesn't match")
if __name__ == '__main__':
unittest.main()
================================================
FILE: aquila/metricstore/test/apis/db_fns.py
================================================
import unittest
import index # includes preloading databases
import router
from utils import CID, schema
class TestDB (unittest.TestCase):
# A fresh DB is created
def test_1_db_fresh_create (self):
schema_def1 = {
"description": "this is my database",
"unique": "r8and0mseEd90",
"encoder": "example.com/autoencoder/API",
"codelen": 3,
"metadata": {
"name": "string",
"age": "number"
}
}
schema_def2 = {
"description": "this is my database",
"unique": "r8and0mseEd90",
"encoder": "example.com/autoencoder/API",
"codelen": 3,
"metadata": {
"name": "string",
"age": "number"
}
}
database_name = router.create_database(schema_def1)
schema_def = schema.generate_schema(schema_def2)
database_name_ = CID.doc2CID(schema_def)
self.assertEqual(database_name, database_name_, "DB name doesn't match")
# An existing DB is created
def test_2_db_exist_create (self):
schema_def1 = {
"description": "this is my database",
"unique": "r8and0mseEd90",
"encoder": "example.com/autoencoder/API",
"codelen": 3,
"metadata": {
"name": "string",
"age": "number"
}
}
schema_def2 = {
"description": "this is my database",
"unique": "r8and0mseEd90",
"encoder": "example.com/autoencoder/API",
"codelen": 3,
"metadata": {
"name": "string",
"age": "number"
}
}
database_name = router.create_database(schema_def1)
schema_def = schema.generate_schema(schema_def2)
database_name_ = CID.doc2CID(schema_def)
self.assertEqual(database_name, database_name_, "DB name doesn't match")
if __name__ == '__main__':
unittest.main()
================================================
FILE: aquila/metricstore/test/apis/doc_fns.py
================================================
import unittest
import index # includes preloading databases
import router
from utils import CID
import numpy as np
import time
class TestDocs (unittest.TestCase):
# A fresh doc is created
def test_1_doc_fresh_create (self):
# create database
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd901",
"encoder": "example.com/autoencoder/API",
"codelen": 3,
"metadata": {
"name": "string",
"age": "number"
}
}
database_name = router.create_database(schema_def)
# add document
docs = [{
"metadata": {
"name":"name1",
"age": 20
},
"code": [1,2,3]
}, {
"metadata": {
"name":"name2",
"age": 30
},
"code": [1,2,3]
}]
cids = router.insert_docs(docs, database_name)
self.assertEqual(len(cids), len(docs), "Document creation failed")
# An incomplete doc is created
def test_1a_doc_incomplete_create (self):
# create database
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd901",
"encoder": "example.com/autoencoder/API",
"codelen": 3,
"metadata": {
"name": "string",
"age": "number"
}
}
database_name = router.create_database(schema_def)
# add document with "code" missing
docs = [{
"metadata": {
"name":"name1",
"age": 20
}
}, {
"metadata": {
"name":"name2",
"age": 30
}
}]
cids = router.insert_docs(docs, database_name)
self.assertEqual(cids, [None, None], "Document creation test failed")
# An existing doc is created
def test_2_doc_exist_create (self):
# create database
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd901",
"encoder": "example.com/autoencoder/API",
"codelen": 3,
"metadata": {
"name": "string",
"age": "number"
}
}
database_name = router.create_database(schema_def)
# add existing document
docs = [{
"metadata": {
"name":"name1",
"age": 20
},
"code": [1,2,3]
}, {
"metadata": {
"name":"name2",
"age": 30
},
"code": [1,2,3]
}]
cids = router.insert_docs(docs, database_name)
self.assertEqual(len(cids), len(docs), "Document creation failed")
# A non existing DB is used to create doc
def test_3_db_fresh_doc_create (self):
# create random DB name
database_name = "BRANDOM"
# add existing document
docs = [{
"metadata": {
"name":"name1",
"age": 20
},
"code": [1,2,3]
}, {
"metadata": {
"name":"name2",
"age": 30
},
"code": [1,2,3]
}]
cids = router.insert_docs(docs, database_name)
self.assertEqual(len(cids), 0, "Document creation failed")
# A fresh doc is deleted
def test_4_doc_fresh_delete (self):
# create database
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd901fr",
"encoder": "example.com/autoencoder/API",
"codelen": 3,
"metadata": {
"name": "string",
"age": "number"
}
}
database_name = router.create_database(schema_def)
# delete non existing documents
cids = router.delete_docs(["sdfsdfsdf", "tret456"], database_name)
self.assertEqual(len(cids), 0, "Doc deletion failed")
# An existing doc is deleted for small dataset
def test_5a_doc_exist_delete_small (self):
# create database
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd902",
"encoder": "example.com/autoencoder/API",
"codelen": 100,
"metadata": {
"name": "string",
"age": "number"
}
}
database_name = router.create_database(schema_def)
# add small epoch document
docs = []
# create special doc
matrix_r_spec = np.random.rand(1, 100)
docs.append({
"metadata": {"name":"special", "age":11},
"code": matrix_r_spec[0].tolist()
})
cids_spec = router.insert_docs(docs, database_name)
# create other docs
for _ in range(90):
docs = []
# create random matrix
matrix_r = np.random.rand(100, 100)
# create documents
for item in matrix_r:
docs.append({
"metadata": {"name":"generic", "age":10},
"code": item.tolist()
})
cids = router.insert_docs(docs, database_name)
# check for doc existance
k = 2
docs, dists = router.search([matrix_r_spec[0].tolist()], k, None, database_name)
self.assertEqual(docs[0][0]["metadata"]["name"], "special", "Doc doesn't exist")
# delete special doc
cids = router.delete_docs(cids_spec, database_name)
time.sleep(10)
# check for doc existance
k = 2
docs, dists = router.search([matrix_r_spec[0].tolist()], k, None, database_name)
self.assertEqual(len(docs[0]), k, "Doc deletion failed")
self.assertEqual(docs[0][0]["metadata"]["name"], "generic", "Doc deletion failed")
# An existing doc is deleted for large dataset
def test_5b_doc_exist_delete_large (self):
# create database
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd902",
"encoder": "example.com/autoencoder/API",
"codelen": 100,
"metadata": {
"name": "string",
"age": "number"
}
}
database_name = router.create_database(schema_def)
# add small epoch document
docs = []
# create special doc
matrix_r_spec = np.random.rand(1, 100)
docs.append({
"metadata": {"name":"special", "age":11},
"code": matrix_r_spec[0].tolist()
})
cids_spec = router.insert_docs(docs, database_name)
# create other docs
for _ in range(120):
docs = []
# create random matrix
matrix_r = np.random.rand(100, 100)
# create documents
for item in matrix_r:
docs.append({
"metadata": {"name":"generic", "age":10},
"code": item.tolist()
})
cids = router.insert_docs(docs, database_name)
time.sleep(60)
# check for doc existance
k = 2
docs, dists = router.search([matrix_r_spec[0].tolist()], k, None, database_name)
self.assertEqual(docs[0][0]["metadata"]["name"], "special", "Doc doesn't exist")
# delete special doc
cids = router.delete_docs(cids_spec, database_name)
time.sleep(60)
# check for doc existance
k = 2
docs, dists = router.search([matrix_r_spec[0].tolist()], k, None, database_name)
self.assertEqual(len(docs[0]), k, "Doc deletion failed")
self.assertEqual(docs[0][0]["metadata"]["name"], "generic", "Doc deletion failed")
# A non existing DB is used to delete doc
def test_6_db_fresh_doc_delete (self):
# create random DB name
database_name = "BRANDOM"
# delete non existing documents
cids = router.delete_docs(["sdfsdfsdf", "tret456"], database_name)
self.assertEqual(len(cids), 0, "Doc deletion failed")
if __name__ == '__main__':
unittest.main()
================================================
FILE: aquila/metricstore/test/apis/search_fns.py
================================================
import unittest
import time
import index # includes preloading databases
import router
from utils import CID
import numpy as np
class TestSearch (unittest.TestCase):
# Perform search on small data
def test_1_insert_and_search_small (self):
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd903",
"encoder": "example.com/autoencoder/API",
"codelen": 100,
"metadata": {}
}
database_name = router.create_database(schema_def)
# insert documents in bulk --------------------
for i_ in range(99):
# create random matrix
matrix_r = np.random.rand(100, 100)
# create documents
docs = []
for item in matrix_r:
docs.append({
"metadata": {},
"code": item.tolist()
})
cids = router.insert_docs(docs, database_name)
k = 10
docs, dists = router.search(np.random.rand(1, 784), k, None, database_name)
self.assertEqual(k, len(docs[0]), "Search failed")
# Perform search on large data
def test_2_insert_and_search_large (self):
schema_def = {
"description": "this is my database",
"unique": "r8and0mseEd904",
"encoder": "example.com/autoencoder/API",
"codelen": 100,
"metadata": {}
}
database_name = router.create_database(schema_def)
# insert documents in bulk --------------------
for i_ in range(120):
# create random matrix
matrix_r = np.random.rand(100, 100)
# create documents
docs = []
for item in matrix_r:
docs.append({
"metadata": {},
"code": item.tolist()
})
cids = router.insert_docs(docs, database_name)
# time.sleep(5)
k = 10
docs, dists = router.search(np.random.rand(1, 784), k, None, database_name)
self.assertEqual(k, len(docs[0]), "Search failed")
if __name__ == '__main__':
unittest.main()
================================================
FILE: aquila/metricstore/utils/CID.py
================================================
import logging
import bson
import hashlib
import base58
def doc2CID (inp):
try:
# JSON OBJ to BSON encode
bson_ = bson.dumps(inp)
# SHA-256 Double Hashing
hash_ = hashlib.sha256(bson_)
hash_ = hashlib.sha256(hash_.digest())
# Convert to Base-58 string
b58c_ = base58.b58encode(hash_.digest())
return b58c_.decode('utf-8')
except Exception as e:
logging.debug(e)
return None
def doc2bson (inp):
try:
# JSON OBJ to BSON encode
bson_ = bson.dumps(inp)
return bson_
except Exception as e:
logging.debug(e)
return None
def bson2doc (inp):
try:
# BSON to JSON OBJ
json_ = bson.loads(inp)
return json_
except Exception as e:
logging.debug(e)
return None
================================================
FILE: aquila/metricstore/utils/config.py
================================================
import yaml
import os
os.environ["DATA_STORE_LOCATION"] = "/data/"
os.environ["LOG_CHUNK_LEN"] = "1000"
if "MINI_AQDB" not in os.environ:
os.environ["MINI_AQDB"] = "inactive"
else:
os.environ["MINI_AQDB"] = "active"
with open("DB_config.yml", "r") as stream:
DB_config = yaml.safe_load(stream)
if "AUTH_KEY_FILE" not in os.environ:
os.environ["AUTH_KEY_FILE"] = str(DB_config["auth"]["pubkey"])
if "FIXED_VEC_DIMENSION" not in os.environ:
os.environ["FIXED_VEC_DIMENSION"] = str(DB_config["docs"]["vd"])
if "MIN_SAWP_COUNT" not in os.environ:
os.environ["MIN_SAWP_COUNT"] = str(DB_config["docs"]["cswap"])
if "FIXED_Q_LEN" not in os.environ:
os.environ["FIXED_Q_LEN"] = str(DB_config["queue"]["qlen"])
if "THREAD_SLEEP" not in os.environ:
os.environ["THREAD_SLEEP"] = str(DB_config["queue"]["sleep"])
if "FAISS_MAX_CELLS" not in os.environ:
os.environ["FAISS_MAX_CELLS"] = str(DB_config["faiss"]["init"]["nlist"])
if "FAISS_VISIT_CELLS" not in os.environ:
os.environ["FAISS_VISIT_CELLS"] = str(DB_config["faiss"]["init"]["nprobe"])
if "FAISS_BYTES_PER_VEC" not in os.environ:
os.environ["FAISS_BYTES_PER_VEC"] = str(DB_config["faiss"]["init"]["bpv"])
if "FAISS_BYTES_PER_SUB_VEC" not in os.environ:
os.environ["FAISS_BYTES_PER_SUB_VEC"] = str(DB_config["faiss"]["init"]["bpsv"])
if "ANNOY_SIM_METRIC" not in os.environ:
os.environ["ANNOY_SIM_METRIC"] = str(DB_config["annoy"]["init"]["smetric"])
if "ANNOY_NTREES" not in os.environ:
os.environ["ANNOY_NTREES"] = str(DB_config["annoy"]["init"]["ntrees"])
if "ANNOY_SK" not in os.environ:
os.environ["ANNOY_SK"] = str(DB_config["annoy"]["init"]["search_k"])
================================================
FILE: aquila/metricstore/utils/cryptops.py
================================================
import logging
from Crypto.Hash import SHA384
from Crypto.PublicKey import RSA
from Crypto.Signature import pkcs1_15
import base58
import chardet
import bson
def read_public_key (location):
# disk read public key
with open(location, "r") as pkf:
k = pkf.read()
pub_key = RSA.import_key(k)
return pub_key
def verify_signature (json_data, pub_key, signature):
ret = True
binary_data = bson.dumps(json_data)
# generate hash
hash = SHA384.new()
hash.update(binary_data)
signature = base58.b58decode(signature)
# Verify with pub key
verifier = pkcs1_15.new(pub_key)
try:
verifier.verify(hash, signature)
except Exception as e:
logging.debug(e)
ret = False
return ret
================================================
FILE: aquila/metricstore/utils/schema.py
================================================
import logging
import fastjsonschema
def generate_schema (template):
# get all keys from template schema
metadata_templ = template.get("metadata")
if metadata_templ:
del template["metadata"]
else:
metadata_templ = {}
keys_ = list(template.keys())
# sort keys
keys_.sort()
# validate required keys
req_k = ["description", "encoder", "unique"]
for r_ in req_k:
if r_ not in keys_:
# cannot continue, invalid schema
return None
# generate schema in sorted keys
generated_schema = {
"$schema": "http://json-schema.org/schema#",
"$id": "http://aquilanetwork.com/schema/id/v1",
"title": "Schema",
"description": "",
"encoder": "",
"unique": "",
"type": "object",
"properties": {
"code": {
"description": "Encoded data",
"type": "array",
"items": {
"type": "number"
}
},
"metadata": {
"$ref": "#/definitions/metadata"
}
},
"definitions": {
"metadata": {
"description": "User defined metadata",
"type": "object",
"properties": {},
"required": []
}
},
"required": ["code", "metadata"]
}
metadata = {}
metadata_types = ["number", "string"]
for key_ in keys_:
if key_ in req_k:
# fill in root level keys
generated_schema[key_] = template[key_]
elif key_ == "codelen":
generated_schema["properties"]["code"]["maxItems"] = template[key_]
else:
# fill in root level keys :: extra info that we don't care
generated_schema[key_] = template[key_]
for key_ in metadata_templ.keys():
# check type is predefined
if metadata_templ[key_] not in metadata_types:
return None
# fill in metadata keys
metadata[key_] = {
"type": metadata_templ[key_]
}
generated_schema["definitions"]["metadata"]["properties"] = metadata
generated_schema["definitions"]["metadata"]["required"] = list(metadata.keys())
# validate values
if type(generated_schema["properties"]["code"].get("maxItems")) != int:
return None
return generated_schema
def compile (schema_def):
# compile schema
validator = fastjsonschema.compile(schema_def)
return validator
def validate_json_docs (validator, json_doc):
try:
# validate doc on schema :: will except on fail
validator(json_doc)
# validated
# logging.debug("Schema validation success")
return True
except Exception as e:
logging.error(e)
return False
================================================
FILE: aquila/metricstore/vec_index/hannoy.py
================================================
import logging
import numpy as np
from annoy import AnnoyIndex
import yaml
import os
import threading
import queue
import time
class Annoy:
def __init__(self, model_location):
# set model location
self.model_location = model_location
# to keep the thread & queue running
self.process_flag = True
self.q_maxsize = int(os.environ["FIXED_Q_LEN"])
self.build_batch_size = int(os.environ["FIXED_Q_LEN"])
self.process_thread = None
self._lock = threading.Lock()
self.process_timeout_sec = int(os.environ["THREAD_SLEEP"]) # seconds
# this is to keep track of all vectors inserted
# for saving into disk and retrieve later
self.index_disk = None
try:
# make sure to parse env variables to their expected type
self.dim = int(os.environ["FIXED_VEC_DIMENSION"])
self.sim_metric = str(os.environ["ANNOY_SIM_METRIC"])
self.n_trees = int(os.environ["ANNOY_NTREES"])
self.search_k = int(os.environ["ANNOY_SK"])
self.model_loaded = self.load_model_from_disk()
if not self.model_loaded:
if self.init_annoy():
logging.debug("Annoy Init done")
else:
logging.debug("Annoy Init Failed")
except Exception as e:
logging.error('Error initializing Annoy: ' + e)
# spawn process thread
self.spawn()
def __del__(self):
self.process_flag = False
if self.process_thread:
self.process_thread.join()
def spawn (self):
# create pipeline to add documents
self.pipeline = queue.Queue(maxsize=self.q_maxsize)
# create process thread
self.process_thread = threading.Thread(target=self.process, args=(), daemon=True)
# start process thread
self.process_thread.start()
# return self.pipeline
def init_annoy(self):
# only do if no index loaded from disk
if not self.model_loaded:
logging.debug('Annoy init index')
self.a_index = AnnoyIndex(self.dim, self.sim_metric)
# Lock index read / wtite until it is built
with self._lock:
# build index
build_ = self.a_index.build(self.n_trees)
if build_:
self.model_loaded = self.save_model_to_disk()
return self.model_loaded
def add_vectors(self, documents):
# add documents to queue
self.pipeline.put({"action":"add", "docs": documents})
return True
def process(self):
while (self.process_flag):
# set a timeout till next vector indexing
time.sleep(self.process_timeout_sec)
# check if queue is not empty
if self.pipeline.qsize() > 0:
# Lock index read / wtite until it is built
with self._lock:
# unbuild index first
self.a_index.unbuild()
len_documents = 0
# fetch all currently available documents from queue
while not self.pipeline.empty():
# extract document & contents
qitem = self.pipeline.get_nowait()
if qitem["action"] == "add":
documents = qitem["docs"]
len_documents += len(documents)
for document in documents:
_id = document["_id"]
vector_e = document["code"]
# add vector to index
self.a_index.add_item(int(_id), vector_e)
# append to disk proxy
if self.index_disk is None:
self.index_disk = np.array([vector_e + [int(_id)]], dtype=float)
else:
self.index_disk = np.append(self.index_disk, [vector_e + [int(_id)]], axis=0)
elif qitem["action"] == "delete":
ids = qitem["ids"]
len_documents += len(ids)
# reset
zero_ = np.zeros(self.dim + 1)
for id_ in ids:
# add zero vector to index
self.a_index.add_item(int(id_), zero_[:-1].tolist())
# reset npy disk array
if self.index_disk is not None:
self.index_disk[ids] = zero_
# take a rest if doc length is > batch_size
if len_documents > self.build_batch_size:
break
# build vector
build_ = self.a_index.build(self.n_trees, n_jobs=-1)
# write to disk
if build_:
self.model_loaded = self.save_model_to_disk()
def delete_vectors(self, ids):
# add documents to queue
self.pipeline.put({"action":"delete", "ids": ids})
return True, ids
def get_nearest_k(self, matrix, k):
ids = []
dists = []
# Lock index read / wtite until nearest neighbor search
with self._lock:
for vec_data in matrix:
if self.search_k != -1:
_id, _dist = self.a_index.get_nns_by_vector(vec_data, k, self.search_k, include_distances=True)
else:
_id, _dist = self.a_index.get_nns_by_vector(vec_data, k, include_distances=True)
ids.append(_id)
dists.append(_dist)
return ids, dists
def get_nearest_rad(self, matrix, rad):
ids = []
dists = []
# Lock index read / wtite until nearest neighbor search
with self._lock:
for vec_data in matrix:
if self.search_k != -1:
_id, _dist = self.a_index.get_nns_by_vector(vec_data, k, self.search_k, include_distances=True)
else:
_id, _dist = self.a_index.get_nns_by_vector(vec_data, k, include_distances=True)
ids.append(_id)
dists.append(_dist)
return ids, dists
def load_model_from_disk(self):
try:
# prepare new index
self.a_index = AnnoyIndex(self.dim, self.sim_metric)
# read index
self.index_disk = np.load(self.model_location+'.npy')
# build Annoy Index
for vec_ in self.index_disk.tolist():
self.a_index.add_item(int(vec_[-1]), vec_[0:-1])
# build index
build_ = self.a_index.build(self.n_trees)
logging.debug('Annoy index loading success')
return True
except Exception as e:
logging.debug('Annoy index loading failed. Creating new index..')
return False
def save_model_to_disk(self):
try:
# write index
np.save(self.model_location, self.index_disk)
logging.debug('Annoy index writing success')
return True
except Exception as e:
logging.error('Annoy index writing failed' + e)
return False
================================================
FILE: aquila/metricstore/vec_index/hfaiss.py
================================================
import logging
import numpy as np
import faiss
import yaml
import os
import threading
import queue
import time
class Faiss:
def __init__(self, model_location):
# set model location
self.model_location = model_location
self.is_active = False
self.nlist = int(os.environ["FAISS_MAX_CELLS"])
self.nprobe = int(os.environ["FAISS_VISIT_CELLS"])
self.bytesPerVec = int(os.environ["FAISS_BYTES_PER_VEC"])
self.bytesPerSubVec = int(os.environ["FAISS_BYTES_PER_SUB_VEC"])
# make sure to parse env variables to their expected type
self.dim = int(os.environ["FIXED_VEC_DIMENSION"])
self.model_loaded = self.load_model_from_disk(self.model_location)
self.is_initiated_ = self.model_loaded
# to keep the thread & queue running
self.process_flag = True
self.q_maxsize = int(os.environ["FIXED_Q_LEN"])
self.process_thread = None
self._lock = threading.Lock()
self.process_timeout_sec = int(os.environ["THREAD_SLEEP"]) # seconds
# spawn process thread
self.spawn()
def __del__(self):
self.process_flag = False
if self.process_thread:
self.process_thread.join()
def spawn (self):
# create pipeline to add documents
self.pipeline = queue.Queue(maxsize=self.q_maxsize)
# create process thread
self.process_thread = threading.Thread(target=self.process, args=(), daemon=True)
# start process thread
self.process_thread.start()
# return self.pipeline
def init_faiss(self, matrix):
self.train_data = np.matrix(matrix).astype('float32')
logging.debug('FAISS init quantizer')
self.f_quantizer = faiss.IndexFlatIP(self.dim)
# Lock index read / wtite until it is built
with self._lock:
logging.debug('FAISS init index')
self.f_index = faiss.IndexIVFPQ(self.f_quantizer, self.dim, self.nlist, self.bytesPerVec, self.bytesPerSubVec)
logging.debug('FAISS train index')
self.f_index.train(self.train_data)
logging.debug('FAISS train index finished')
# write index to disk
self.model_loaded = self.save_model_to_disk(self.model_location, self.f_index)
self.is_initiated_ = self.model_loaded
return self.is_initiated_
def is_initiated(self):
return self.is_initiated_
def load_model_from_disk(self, location):
try:
# read index
self.f_index = faiss.read_index(location)
logging.debug('FAISS index loading success')
return True
except Exception as e:
logging.error('FAISS index loading failed' + str(e))
return False
def save_model_to_disk(self, location, index):
try:
# write index
faiss.write_index(index, location)
logging.debug('FAISS index writing success')
return True
except:
logging.error('FAISS index writing failed')
return False
def add_vectors(self, documents):
# add document to queue
self.pipeline.put({"action":"add", "docs": documents})
return True
def process(self):
while (self.process_flag):
# set a timeout till next vector indexing
time.sleep(self.process_timeout_sec)
# check if queue is not empty
if self.pipeline.qsize() > 0:
ids = []
vecs = []
# f_delete = False
# f_add = False
# fetch all currently available documents from queue
while not self.pipeline.empty():
# extract document & contents
qitem = self.pipeline.get_nowait()
if qitem["action"] == "add":
# f_add = True
documents = qitem["docs"]
for document in documents:
_id = document["_id"]
vector_e = document["code"]
ids.append(_id)
vecs.append(vector_e)
elif qitem["action"] == "delete":
# f_delete = True
ids = qitem["ids"]
# remove vectors and add zero reset
self.f_index.remove_ids(np.array(ids).astype('int'))
vecs = np.zeros((len(ids), self.dim))
# if f_add:
# convert to np matrix
vec_data = np.matrix(vecs).astype('float32')
id_data = np.array(ids).astype('int')
# Lock index read / wtite until it is built
with self._lock:
# add vectors
self.f_index.add_with_ids(vec_data, id_data)
# if f_delete:
# pass
# write to disk
self.save_model_to_disk(self.model_location, self.f_index)
def delete_vectors(self, ids):
# remove vectors
self.pipeline.put({"action":"delete", "ids": ids})
return True, ids
def get_nearest_rad(self, matrix, rad):
pass
# # convert to np matrix
# vec_data = np.matrix(matrix).astype('float32')
# # Lock index read / wtite until nearest neighbor search
# with self._lock:
# D, I = self.f_index.search(vec_data, k)
# return I.tolist(), D.tolist()
def get_nearest_k(self, matrix, k):
# convert to np matrix
vec_data = np.matrix(matrix).astype('float32')
# Lock index read / wtite until nearest neighbor search
with self._lock:
D, I = self.f_index.search(vec_data, k)
return I.tolist(), D.tolist()
================================================
FILE: aquila/metricstore/wal/walman.py
================================================
import logging
import os
import time
import queue
import threading
LOG_STORE_LOCATION = os.environ["DATA_STORE_LOCATION"] + "/log/"
LOG_CHUNK_LEN = int(os.environ["LOG_CHUNK_LEN"])
PROCESS_TIMEOUT = int(os.environ["THREAD_SLEEP"])
MAX_Q_LEN = int(os.environ["FIXED_Q_LEN"])
process_flag = True
pipeline = None
process_thread = None
# list logs in log directory
log_files_asc = os.listdir(LOG_STORE_LOCATION).sort()
# load last log chunk
last_chunk_name = log_files_asc[-1]
last_chunk_data_len = 0
# read last chunk
with open(LOG_STORE_LOCATION + last_chunk_name) as flog:
# check if the file is full capacity
if len(flog.split("\n")) >= LOG_CHUNK_LEN:
create_new_chunk()
else:
last_chunk_data_len = len(flog.split("\n"))
def create_new_chunk ():
global log_files_asc
global last_chunk_name
global last_chunk_data_len
# name a new log file
new_f_name = time.time()
log_files_asc.append(new_f_name)
last_chunk_name = new_f_name
# initialize log file
with open(LOG_STORE_LOCATION + last_chunk_name, "a") as aflog:
aflog.write("WALOG INIT")
last_chunk_data_len = 1
def spawn ():
# spawn worker thread
global process_flag
process_flag = True
# create pipeline to add documents
global pipeline
pipeline = queue.Queue(maxsize=MAX_Q_LEN)
# create process thread
global process_thread
process_thread = threading.Thread(target=process, args=(), daemon=True)
# start process thread
process_thread.start()
def add_log (batch_in):
global pipeline
pipeline.put(batch_in)
return True
def terminate ():
global process_flag
process_flag = False
def process ():
global pipeline
while (process_flag):
time.sleep(PROCESS_TIMEOUT)
# check if queue is not empty
if not pipeline.empty():
# fetch all currently available codes from queue
while not pipeline.empty():
# pop available logs
for batch in pipeline.get_nowait():
for log_ in batch:
# check if the batch length exceeded
if last_chunk_data_len + 1 >= LOG_CHUNK_LEN:
create_new_chunk()
# write log to disk
with open(LOG_STORE_LOCATION + last_chunk_name, "a") as aflog:
aflog.write("\n" + log_)
================================================
FILE: aquila/metricstore/wal/walmon.py
================================================
================================================
FILE: aquila/network/go.mod
================================================
module aquilaport
go 1.15
require (
github.com/gorilla/mux v1.8.0
github.com/syndtr/goleveldb v1.0.0
go.mongodb.org/mongo-driver v1.4.4
)
================================================
FILE: aquila/network/go.sum
================================================
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
go.mongodb.org/mongo-driver v1.4.4 h1:bsPHfODES+/yx2PCWzUYMH8xj6PVniPI8DQrsJuSXSs=
go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
================================================
FILE: aquila/network/main.go
================================================
// main.go
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/cookiejar"
"strings"
"time"
"github.com/gorilla/mux"
"github.com/syndtr/goleveldb/leveldb"
"go.mongodb.org/mongo-driver/bson"
)
var _localDb, _ = leveldb.OpenFile("./db/_local", nil)
var replicationDb, _ = leveldb.OpenFile("./db/replication", nil)
var sourceDb, _ = leveldb.OpenFile("./db/source", nil)
var jar, err = cookiejar.New(nil)
// if err != nil {
// fmt.Println(err)
// }
// Document struct
type Document struct {
ID string `json:"id"`
Title string `json:"title"`
Deleted bool `json:"deleted"`
Timestamp string `json:"timestamp"`
Version string `json:"version"`
}
func homePage(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Welcome to the HomePage!")
fmt.Println("Endpoint Hit: homePage")
}
func createNewDocument(w http.ResponseWriter, r *http.Request) {
// decode json body
var documents []Document
json.NewDecoder(r.Body).Decode(&documents)
// init a batch insert to level
batch := new(leveldb.Batch)
// insert docs to batch
for _, doc := range documents {
// update document version
doc.Version = string(getVersion(doc))
// convert struct to bson
data, err := bson.Marshal(doc)
fmt.Println(err)
// insert doc
batch.Put([]byte(doc.ID), data)
}
// write batch to level db
err := sourceDb.Write(batch, nil)
fmt.Println(err)
// iterate over leveldb and get key, val
iter := sourceDb.NewIterator(nil, nil)
for iter.Next() {
key := iter.Key()
value := iter.Value()
var docRet Document
// convert bson to byte
bson.Unmarshal(value, &docRet)
fmt.Println(string(key), docRet)
}
iter.Release()
err = iter.Error()
json.NewEncoder(w).Encode(documents)
}
func deleteDocument(w http.ResponseWriter, r *http.Request) {
var ids []string
json.NewDecoder(r.Body).Decode(&ids)
// init a batch insert to level
batch := new(leveldb.Batch)
// delete docs batch
for _, id := range ids {
// delete doc
batch.Delete([]byte(id))
}
// write batch to level db
err := sourceDb.Write(batch, nil)
fmt.Println(err)
// iterate over leveldb and get key, val
iter := sourceDb.NewIterator(nil, nil)
for iter.Next() {
key := iter.Key()
value := iter.Value()
var docRet Document
// convert bson to byte
bson.Unmarshal(value, &docRet)
fmt.Println(string(key), docRet)
}
iter.Release()
err = iter.Error()
json.NewEncoder(w).Encode(ids)
}
func getDocuments(selector string) []Document {
var documents []Document
if selector == "all" {
// iterate over leveldb and get key, val
iter := sourceDb.NewIterator(nil, nil)
for iter.Next() {
// key := iter.Key()
value := iter.Value()
var docRet Document
// convert bson to byte
bson.Unmarshal(value, &docRet)
documents = append(documents, docRet)
}
iter.Release()
err = iter.Error()
}
return documents
}
func getVersion(document Document) []byte {
// version: timestamp (milliseconds, 13 digits) + deleted
var delStatus byte
delStatus = 48
if document.Deleted {
delStatus = 49
}
versionGen := append([]byte(document.Timestamp), delStatus)
return versionGen
}
// =========================== COUCHDB ======================================================================
func authenticate() (int, []byte) {
return request("http://127.0.0.1:5984/_session", "POST", "name=admin&password=password", "x-www-form-urlencoded")
}
func checkDB(dbName string) (int, []byte) {
return request("http://127.0.0.1:5984/"+dbName, "HEAD", "", "")
}
func createDB(dbName string) (int, []byte) {
return request("http://127.0.0.1:5984/"+dbName, "PUT", "", "")
}
func getDBInfo(dbName string) (int, []byte) {
return request("http://127.0.0.1:5984/"+dbName, "GET", "", "")
}
func getReplicationLog(dbName string, logID string) (int, []byte) {
return request("http://127.0.0.1:5984/"+dbName+"/_local/"+logID, "GET", "", "")
}
func addBatchDocs(dbName string, documents []Document) (int, []byte) {
data, err := json.Marshal(documents)
if err != nil {
fmt.Println(err)
}
dataStr := `{"docs":` + string(data) + `}`
return request("http://127.0.0.1:5984/"+dbName+"/_bulk_docs", "POST", dataStr, "application/json")
}
func getChanges(dbName string) (int, []byte) {
return request("http://127.0.0.1:5984/"+dbName+"/_changes?style=all_docs", "GET", "", "")
}
func ensureCommit(dbName string) (int, []byte) {
return request("http://127.0.0.1:5984/"+dbName+"/_ensure_full_commit", "POST", "", "application/json")
}
func setReplChkPoint(dbName string, replLog []byte) (int, []byte) {
return request("http://127.0.0.1:5984/"+dbName+"/_ensure_full_commit", "POST", string(replLog), "application/json")
}
func request(url string, method string, payload string, contentType string) (int, []byte) {
client := &http.Client{
Jar: jar,
}
var req *http.Request
var err error
if payload == "" {
req, err = http.NewRequest(method, url, nil)
} else {
req, err = http.NewRequest(method, url, strings.NewReader(payload))
}
if err != nil {
fmt.Println(err)
}
if contentType == "x-www-form-urlencoded" {
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
} else if contentType == "application/json" {
req.Header.Add("Content-Type", "application/json")
}
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
}
return res.StatusCode, body
}
func handleRequests() {
myRouter := mux.NewRouter().StrictSlash(true)
myRouter.HandleFunc("/", homePage)
myRouter.HandleFunc("/create", createNewDocument).Methods("POST")
myRouter.HandleFunc("/delete", deleteDocument).Methods("POST")
log.Fatal(http.ListenAndServe(":10000", myRouter))
}
func replicatorDemon() {
for {
// authenticate couchDB
stat, _ := authenticate()
if stat != 200 {
fmt.Println("CouchDB Authentication failed. Exiting demon.")
} else {
fmt.Println("CouchDB authentication success.")
}
// perform replication to target =======================================
// sourceDB := "source"
targetDB := "target"
// 1. verify peers
stat, _ = checkDB(targetDB)
if stat == 404 {
fmt.Println("Target DB do not exist. Creating it..")
// create target database
stat, _ = createDB(targetDB)
if stat == 201 {
fmt.Println("Target DB created.")
} else {
fmt.Println("Target DB can not be created. Aborting replication..")
break
}
} else if stat == 200 {
fmt.Println("Target DB exists")
}
// 2. get peers information
fmt.Println("Getting peers information..")
stat, _ = getDBInfo(targetDB)
replicationID := ""
if stat == 200 {
// generate replication ID
replicationID = "123456"
}
// 3. find common ascestry
fullReplication := false
if replicationID != "" {
fmt.Println("Generated replication ID: ", replicationID)
// get replication log from target
fmt.Println("Getting replication log from target..")
stat, rlog := getReplicationLog(targetDB, replicationID)
if stat == 200 {
// compare logs
fmt.Println(string(rlog))
} else if stat == 404 {
fullReplication = true
fmt.Println("Replication log not available. Full replication needed.")
}
} else {
fmt.Println("Replication ID generation failed. Exiting.. ")
break
}
// 4. locate changed documents
var documents []Document
if fullReplication {
// get all documents
documents = getDocuments("all")
} else {
// Get changed documents in target
stat, changes := getChanges(targetDB)
if stat == 200 {
fmt.Println("Changes: ", changes)
}
// finalize documents to be replicated
documents = getDocuments("all") // TODO: To be changed to selectives
// finalize replication if no change found
if len(documents) <= 0 {
fmt.Println("No more changes to replicate.")
}
}
// 5. replicate changes
if len(documents) > 0 {
stat, _ := addBatchDocs(targetDB, documents)
if stat == 201 {
fmt.Println("Documents written succesfully.")
} else {
fmt.Println("Documents write failed.")
}
// ensure in commit
stat, data := ensureCommit(targetDB)
if stat == 201 {
fmt.Println("Documents commited succesfully.")
} else {
fmt.Println("Documents commit failed.", string(data))
break
}
// set record replication checkpoint
stat, _ = setReplChkPoint(targetDB, []byte(""))
}
// wait 5 seconds before next replication loop
time.Sleep(time.Duration(5 * time.Second))
}
}
func main() {
go replicatorDemon()
handleRequests()
}
================================================
FILE: aquila/search/.dockerignore
================================================
node_modules
.vscode
.env*
dist
================================================
FILE: aquila/search/.nvmrc
================================================
v16.17.0
================================================
FILE: aquila/search/Dockerfile
================================================
FROM node:16-alpine as builder
# Browserless
ARG BROWSERLESS_API_KEY=your-api-key-here
WORKDIR /app
RUN apk add curl bash
RUN apk add --no-cache git openssh
# install node-prune (https://github.com/tj/node-prune)
RUN curl -sfL https://gobinaries.com/tj/node-prune | bash -s -- -b /usr/local/bin
COPY . .
RUN yarn
RUN yarn build
RUN npm prune --production
# run node prune
RUN /usr/local/bin/node-prune
FROM node:16-alpine
# Application Port
ENV PORT=5000
# Database
ENV DB_HOST=host.docker.internal
ENV DB_PORT=5432
ENV DB_NAME=test_db
ENV DB_USERNAME=user
ENV DB_PASSWORD=password
# Redis
ENV REDIS_HOST=redis-19166.c62.us-east-1-4.ec2.cloud.redislabs.com
ENV REDIS_PORT=19166
ENV REDIS_USERNAME=default
ENV REDIS_PASSWORD=WUFOqcuORH4VoVNKBmzPIQvwsmpHqE5a
# Jwt
ENV JWT_SECRET=jwtsecret
ENV JWT_EXPIRES_IN=2h
#Summarizer
ENV SUMMARIZER_URL=http://host.docker.internal:5008/process
#Aquila
ENV AQUILA_DB_HOST=http://host.docker.internal
ENV AQUILA_DB_PORT=5001
ENV AQUILA_DB_KEY_PATH=../../private_unencrypted.pem
ENV AQUILA_HUB_HOST=http://host.docker.internal
ENV AQUILA_HUB_PORT=5002
ENV AQUILA_HUB_KEY_PATH=../../private_unencrypted_hub.pem
# Browserless
ENV BROWSERLESS_API_KEY=b94a8c0c-3b5c-4342-ab2d-a4b19bf1f13b
WORKDIR /app
COPY --from=builder /app/dist ./dist
COPY --from=builder /app/node_modules ./node_modules
EXPOSE 5000
CMD ["node", "./dist/index.js"]
================================================
FILE: aquila/search/README.md
================================================
# AquilaX EE
## ⚠️ Server and other Credentials are in workflow file
Never make this repo Public
Steps to run this project:
1. Run `yarn` command
2. Setup database settings inside `data-source.ts` file
3. Run `yarn run start:dev` command
4. Generate migration `yarn run typeorm -- migration:generate src/migrations/bookmark --dataSource src/config/db.ts`
5. Run migration `yarn run typeorm -- migration:run --dataSource src/config/db.ts`
## Run using Docker
Docker build
```bash
docker build -t manekshms/aquila-network-api .
docker run -d -p 5000:5000 \
-e DB_HOST=localhost \
-e DB_PORT=5432 \
-e DB_NAME=test_db \
-e DB_USERNAME=test \
-e DB_PASSWORD=example \
-e REDIS_HOST=localhost \
-e REDIS_PORT=6379 \
-e REDIS_USERNAME=default \
-e REDIS_PASSWORD=redis123 \
-e JWT_SECRET=jwtsecret \
-e JWT_EXPIRES_IN=2h \
-e SUMMARIZER_URL=http://localhost:5008/process \
-e AQUILA_DB_HOST=http://localhost.internal \
-e AQUILA_DB_PORT=5001 \
-e AQUILA_DB_KEY_PATH=/ossl/private_unencrypted.pem \
-e AQUILA_HUB_HOST=http://localhost.internal \
-e AQUILA_HUB_PORT=5002 \
-e AQUILA_HUB_KEY_PATH=/ossl/private_unencrypted.pem \
-e ENV BROWSERLESS_API_KEY=key \
-v ${HOME}/aquilax/ossl:/ossl
--name manekshms/aquila-network-ui
```
================================================
FILE: aquila/search/package.json
================================================
{
"name": "aquilax-ee",
"version": "1.0.0",
"description": "AquilaX api",
"main": "index.js",
"repository": "https://github.com/Aquila-HQ/AquilaX-EE.git",
"author": "manekshms <manekshms@gmail.com>",
"scripts": {
"build": "tsc",
"listen": "node dist/index.js",
"start": "yarn listen",
"dev": "cross-env NODE_ENV=development nodemon -e './**/*.ts' --exec 'yarn build && yarn listen'",
"typeorm": "typeorm-ts-node-commonjs"
},
"license": "MIT",
"devDependencies": {
"@types/body-parser": "^1.19.2",
"@types/cors": "^2.8.13",
"@types/express": "^4.17.17",
"@types/jsonwebtoken": "^9.0.1",
"@types/multer": "^1.4.7",
"@types/node": "^18.15.0",
"@types/uuid": "^9.0.1",
"cross-env": "^7.0.3",
"nodemon": "^2.0.21",
"ts-node": "^10.9.1",
"typescript": "^4.9.5"
},
"dependencies": {
"@bull-board/express": "^5.0.0",
"@postlight/parser": "^2.2.3",
"aquila-js": "^1.0.3",
"axios": "^1.3.4",
"body-parser": "^1.20.2",
"bs58": "^5.0.0",
"bullmq": "^3.10.1",
"cheerio": "^1.0.0-rc.12",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cors": "^2.8.5",
"express": "^4.18.2",
"express-validator": "^6.15.0",
"ioredis": "^5.3.1",
"jsonwebtoken": "^9.0.0",
"multer": "^1.4.5-lts.1",
"pg": "^8.10.0",
"puppeteer-core": "^19.7.4",
"reflect-metadata": "^0.1.13",
"routing-controllers": "^0.10.2",
"typedi": "^0.10.0",
"typeorm": "^0.3.12",
"uuid": "^9.0.0"
}
}
================================================
FILE: aquila/search/src/@types/express/index.d.ts
================================================
import { AccountStatus } from "../../service/dto/AuthServiceDto";
declare global {
namespace Express {
interface Request {
token?: string;
jwtTokenPayload?: {
customerId: string;
accountStatus: AccountStatus
}
}
}
}
================================================
FILE: aquila/search/src/@types/index.d.ts
================================================
declare module '@postlight/parser' {
export type ExtractorArgs = any;
export type ParserOptions = {
html?: string;
fetchAllPages?: boolean;
fallback?: boolean;
contentType?: 'html' | 'markdown' | 'text';
headers?: Record<string, string>;
extend?: boolean;
customExtractor?: (args: ExtractorArgs) => ParserResult;
};
export type ParserResult = {
title: string;
content: string;
author: string;
date_published: string;
lead_image_url: string;
dek: string;
next_page_url: string;
url: string;
domain: string;
excerpt: string;
word_count: number;
direction: string;
total_pages: number;
rendered_pages: number;
};
export function parse(
url: string,
opts: ParserOptions
): Promise<ParserResult>;
const exported: {
parse: typeof parse;
};
export default exported;
}
================================================
FILE: aquila/search/src/app.ts
================================================
import 'reflect-metadata';
import express from 'express';
import { Container } from 'typedi';
import { useExpressServer, useContainer} from 'routing-controllers';
import { BullAdapter } from '@bull-board/api/bullAdapter';
import { ExpressAdapter } from '@bull-board/express';
import { createBullBoard } from '@bull-board/api';
import db from './config/db';
import redisConnection from './config/redisConnection';
import { AquilaClientService } from './lib/AquilaClientService';
import { authorizationChecker } from './helper/auth';
import { currentUserChecker } from './helper/user';
import { getAppWorker } from './job/appWorker';
import { AppQueue } from './job/AppQueue';
export default async function main() {
useContainer(Container);
const app = express();
app.use(express.json());
useExpressServer(app, {
cors: true,
authorizationChecker,
currentUserChecker,
middlewares: [`${__dirname}/middleware/**/*.{ts,js}`],
controllers: [`${__dirname}/controller/*.js`],
defaultErrorHandler: false
});
await db.initialize();
const appWorker = getAppWorker(redisConnection);
const aqc = Container.get(AquilaClientService);
await aqc.connect();
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath('/admin/queues');
const appQueue = Container.get(AppQueue);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [new BullAdapter(appQueue.queue)],
serverAdapter: serverAdapter,
});
app.use('/admin/queues', serverAdapter.getRouter());
return app;
}
================================================
FILE: aquila/search/src/config/db.ts
================================================
import { DataSource, DataSourceOptions } from "typeorm";
import { ConfigService } from "../lib/ConfigService";
const configService = new ConfigService();
const dataSourceOptions: DataSourceOptions = {
type: 'postgres',
host: configService.get('DB_HOST'),
port: parseInt(configService.get('DB_PORT'), 10),
username: configService.get('DB_USERNAME'),
password: configService.get('DB_PASSWORD'),
database: configService.get('DB_NAME'),
synchronize: false,
entities: [`${__dirname}/../entity/**/*.{ts,js}`],
migrations: [`${__dirname}/../migrations/**/*.{ts,js}`],
migrationsTableName: "migrations",
}
const dataSource = new DataSource(dataSourceOptions);
export default dataSource;
================================================
FILE: aquila/search/src/config/redisConnection.ts
================================================
import Redis from 'ioredis';
import { ConfigService } from '../lib/ConfigService';
const configService = new ConfigService();
const redisConnection = new Redis({
host: configService.get('REDIS_HOST'),
port: parseInt(configService.get('REDIS_PORT'), 10),
username: configService.get('REDIS_USERNAME'),
password: configService.get('REDIS_PASSWORD')
})
export default redisConnection;
================================================
FILE: aquila/search/src/controller/AuthController.ts
================================================
import { Body, JsonController, Post } from "routing-controllers";
import { Service } from "typedi";
import { AuthService } from "../service/AuthService";
import { LoginCustomerRequestBodyDto, LoginCustomerResponseBodyDto } from "./dto/AuthControllerDto";
@Service()
@JsonController('/auth')
export class AuthController {
public constructor(private authService: AuthService) {}
@Post('/login')
public async login(
@Body() body: LoginCustomerRequestBodyDto
): Promise<LoginCustomerResponseBodyDto> {
const token = await this.authService.login(body.secretKey);
return {
token
}
}
}
================================================
FILE: aquila/search/src/controller/BookmarkController.ts
================================================
import { Body, Get, JsonController, Param, Post, QueryParams, UseBefore } from "routing-controllers";
import { Service } from "typedi";
import { Bookmark } from "../entity/Bookmark";
import { BookmarkTemp } from "../entity/BookmarkTemp";
import { JwtPayloadData } from "../helper/decorators/jwtPayloadData";
import { AuthMiddleware } from "../middleware/global/AuthMiddleware";
import { AddBookmarkValidator } from "../middleware/validator/bookmark/AddBookmarkValidator";
import { GetBookmarkByCollectionIdValidator } from "../middleware/validator/bookmark/GetBookmarkByCollectionIdValidator";
import { GetFeaturedBookmarkValidator } from "../middleware/validator/bookmark/GetFeaturedBookmarkValidator";
import { GetPublicBookmarkByCollectionIdParamValidator } from "../middleware/validator/bookmark/GetPublicBookmarkByCollectionIdParamValidator";
import { GetPublicBookmarkByCollectionIdValidator } from "../middleware/validator/bookmark/GetPublicBookmarkByCollectionIdValidator";
import { BookmarkService } from "../service/BookmarkService";
import { AccountStatus, JwtPayload } from "../service/dto/AuthServiceDto";
import { GetBookmarksByCollectionIdOptionsInputDto, GetFeaturedBookmarksOptionsInputDto } from "../service/dto/BookmarkServiceDto";
import { AddBookmarkReqBodyDto, GetBookmarksByCollectionIdReqQueryParamsDto, GetBookmarksByCollectionIdResBodyDto, GetFeaturedBookmarksReqQueryParamsDto, GetFeaturedBookmarksResBodyDto } from "./dto/BookmarkControllerDto";
@Service()
@JsonController('/bookmark')
export class BookmarkController {
public constructor(private bookmarkService: BookmarkService) {}
@UseBefore(AuthMiddleware, AddBookmarkValidator)
@Post('/')
public async addBookmark(
@Body() body: AddBookmarkReqBodyDto,
@JwtPayloadData() JwtPayloadData: JwtPayload
): Promise<Bookmark|BookmarkTemp> {
return await this.bookmarkService.addBookmark(body, JwtPayloadData.accountStatus);
}
@UseBefore(AuthMiddleware, GetBookmarkByCollectionIdValidator)
@Get('/:collectionId/search')
public async getBookmarksByCollectionId(
@Param('collectionId') collectionId: string,
@QueryParams() queryParams: GetBookmarksByCollectionIdReqQueryParamsDto,
@JwtPayloadData() JwtPayloadData: JwtPayload
): Promise<GetBookmarksByCollectionIdResBodyDto> {
const options: GetBookmarksByCollectionIdOptionsInputDto = {
limit: queryParams.limit ? parseInt(queryParams.limit, 10) : 10,
page: queryParams.page ? parseInt(queryParams.page, 10) : 1
}
if(queryParams.query) {
options.query = queryParams.query;
}
return this.bookmarkService.getBookmarksByCollectionId(collectionId, options, JwtPayloadData.accountStatus);
}
@UseBefore(GetPublicBookmarkByCollectionIdValidator, GetPublicBookmarkByCollectionIdParamValidator)
@Get('/public/:collectionId/search')
public async getPublicBookmarksByCollectionId(
@Param('collectionId') collectionId: string,
@QueryParams() queryParams: GetBookmarksByCollectionIdReqQueryParamsDto,
): Promise<GetBookmarksByCollectionIdResBodyDto> {
const options: GetBookmarksByCollectionIdOptionsInputDto = {
limit: queryParams.limit ? parseInt(queryParams.limit, 10) : 10,
page: queryParams.page ? parseInt(queryParams.page, 10) : 1
}
if(queryParams.query) {
options.query = queryParams.query;
}
return this.bookmarkService.getBookmarksByCollectionId(collectionId, options, AccountStatus.PERMANENT);
}
@UseBefore(GetFeaturedBookmarkValidator)
@Get('/public/featured')
public async getAllFeaturedBookmark(
@QueryParams() queryParams: GetFeaturedBookmarksReqQueryParamsDto
): Promise<GetFeaturedBookmarksResBodyDto> {
const options: GetFeaturedBookmarksOptionsInputDto = {
limit: queryParams.limit ? parseInt(queryParams.limit, 10) : 10,
page: queryParams.page ? parseInt(queryParams.page, 10) : 1
}
return this.bookmarkService.getFeaturedBookmarks(options);
}
}
================================================
FILE: aquila/search/src/controller/CollectionController.ts
================================================
import { Authorized, Get, JsonController, Param, QueryParams, UseBefore } from "routing-controllers";
import { Service } from "typedi";
import { Collection } from "../entity/Collection";
import { CollectionTemp } from "../entity/CollectionTemp";
import { JwtPayloadData } from "../helper/decorators/jwtPayloadData";
import { GetAllFeaturedCollectionValidator } from "../middleware/validator/collection/GetAllFeaturedCollectionValidator";
import { GetAllPublicCollectionValidator } from "../middleware/validator/collection/GetAllPublicCollectionValidator";
import { GetPublicCollectionByIdValidator } from "../middleware/validator/collection/GetPublicCollectionByIdValidator";
import { CollectionService } from "../service/CollectionService";
import { AccountStatus, JwtPayload } from "../service/dto/AuthServiceDto";
import { GetAllFeaturedCollectionReqQueryParamsDto, GetAllFeaturedCollectionResPayloadDto, GetAllPublicCollectionReqQueryParamsDto, GetAllPublicCollectionResPayloadDto } from "./dto/CollectionControllerDto";
@Service()
@JsonController('/collection')
export class CollectionController {
public constructor(private collectionService: CollectionService) {}
@UseBefore(GetAllPublicCollectionValidator)
@Get('/public')
public async getAllPublicCollection(
@QueryParams() queryParams: GetAllPublicCollectionReqQueryParamsDto
): Promise<GetAllPublicCollectionResPayloadDto> {
const options = {
limit: queryParams.limit? parseInt(queryParams.limit) : 10,
page: queryParams.page ? parseInt(queryParams.page) : 1,
where: {
isShareable: true
}
};
return await this.collectionService.getAllCollections(options, AccountStatus.PERMANENT);
}
@UseBefore(GetAllFeaturedCollectionValidator)
@Get('/public/featured')
public async getAllPublicFeaturedCollection(
@QueryParams() queryParams: GetAllFeaturedCollectionReqQueryParamsDto
): Promise<GetAllFeaturedCollectionResPayloadDto> {
const options = {
limit: queryParams.limit? parseInt(queryParams.limit) : 10,
page: queryParams.page ? parseInt(queryParams.page) : 1,
where: {
isShareable: true,
isFeatured: true
}
};
return await this.collectionService.getAllCollections(options, AccountStatus.PERMANENT);
}
@UseBefore(GetPublicCollectionByIdValidator)
@Get('/public/:collectionId')
public async getPublicCollectionById(
@Param('collectionId') collectionId: string
): Promise<Collection> {
return await this.collectionService.getPublicCollectionById(collectionId);
}
@Authorized()
@Get('/my-collections')
public async getCurrentCustomerCollections(
@JwtPayloadData() jwtPayload: JwtPayload
): Promise<Collection[] | CollectionTemp[]> {
const collections = await this.collectionService.getCustomerCollectionsByCustomerId(jwtPayload.customerId, jwtPayload.accountStatus);
return collections;
}
}
================================================
FILE: aquila/search/src/controller/CollectionSubscriptionController.ts
================================================
import { Body, Get, JsonController, Param, Post, QueryParams, UseBefore } from "routing-controllers";
import { Service } from "typedi";
import { Collection } from "../entity/Collection";
import { CollectionSubscription } from "../entity/CollectionSubscription";
import { CollectionSubscriptionTemp } from "../entity/CollectionSubscriptionTemp";
import { JwtPayloadData } from "../helper/decorators/jwtPayloadData";
import { AuthMiddleware } from "../middleware/global/AuthMiddleware";
import { SubscribeCollectionValidator } from "../middleware/validator/csubscription/SubscribeCollectionValidator";
import { UnSubscribeCollectionValidator } from "../middleware/validator/csubscription/UnSubscribeCollectionValidator";
import { CollectionSubscriptionService } from "../service/CollectionSubscriptionService";
import { JwtPayload } from "../service/dto/AuthServiceDto";
import { GetSubscriptionsByCustomerIdOptionsInputDto } from "../service/dto/CollectionSubscriptionServiceDto";
import { GetSubscriptionsByCustomerIdReqQueryParamsDto, GetSubscriptionsByCustomerIdResBodyDto } from "./dto/CollectionSubscriptionControllerDto";
@Service()
@JsonController('/subscription')
export class CollectionSubscriptionController {
public constructor(private collectionSubscriptionService: CollectionSubscriptionService) {}
@UseBefore(AuthMiddleware)
@Get('/list')
public async getCurrentCustomerSubscriptions(
@JwtPayloadData() JwtPayloadData: JwtPayload
): Promise<CollectionSubscriptionTemp[] | CollectionSubscription[]> {
return await this.collectionSubscriptionService.getCustomerSubscriptions(JwtPayloadData.customerId, JwtPayloadData.accountStatus);
}
@UseBefore(AuthMiddleware, SubscribeCollectionValidator)
@Post('/:collectionId/add')
public async subscribeCollection(
@Param('collectionId') collectionId: string,
@JwtPayloadData() JwtPayloadData: JwtPayload
): Promise<CollectionSubscription|CollectionSubscriptionTemp> {
return await this.collectionSubscriptionService.subscribeCollection(collectionId, JwtPayloadData.customerId, JwtPayloadData.accountStatus);
}
@UseBefore(AuthMiddleware)
@Post('/:collectionId/is-subscribed')
public async isCollectionAlreadySubscribed(
@Param('collectionId') collectionId: string,
@JwtPayloadData() jwtPayloadData: JwtPayload
):Promise<{ isSubscribed: boolean}> {
const isSubscribed = await this.collectionSubscriptionService.isCollectionSubscribedByCustomer(collectionId, jwtPayloadData.customerId, jwtPayloadData.accountStatus);
return { isSubscribed };
}
@UseBefore(AuthMiddleware, UnSubscribeCollectionValidator)
@Post('/:collectionId/remove')
public async unSubscribeCollection(
@Param('collectionId') collectionId: string,
@JwtPayloadData() JwtPayloadData: JwtPayload
): Promise<CollectionSubscription | CollectionSubscriptionTemp | null> {
return await this.collectionSubscriptionService.unSubscribeCollection(collectionId, JwtPayloadData.customerId, JwtPayloadData.accountStatus);
}
@Get('/')
@UseBefore(AuthMiddleware)
public async getSubscriptionsByCustomerId(
@JwtPayloadData() jwtPayloadData: JwtPayload,
@QueryParams() queryParams: GetSubscriptionsByCustomerIdReqQueryParamsDto,
): Promise<GetSubscriptionsByCustomerIdResBodyDto> {
const options: GetSubscriptionsByCustomerIdOptionsInputDto = {
limit: queryParams.limit ? parseInt(queryParams.limit, 10) : 10,
page: queryParams.page ? parseInt(queryParams.page, 10) : 1
}
if(queryParams.query) {
options.query = queryParams.query;
}
return await this.collectionSubscriptionService.getSubscriptionsByCustomerId(jwtPayloadData.customerId, options, jwtPayloadData.accountStatus);
}
@UseBefore(AuthMiddleware)
@Get('/collections')
public async getCustomerSubscribedCollections(
@JwtPayloadData() jwtPayloadData: JwtPayload,
): Promise<Collection[]> {
return await this.collectionSubscriptionService.getCustomerSubscribedCollections(jwtPayloadData.customerId, jwtPayloadData.accountStatus);
}
}
================================================
FILE: aquila/search/src/controller/CustomerController.ts
================================================
import { Authorized, Body, CurrentUser, Get, JsonController, Param, Patch, Post, Put, UseBefore } from "routing-controllers";
import { Service } from "typedi";
import { Customer } from "../entity/Customer";
import { CustomerTemp } from "../entity/CustomerTemp";
import { JwtPayloadData } from "../helper/decorators/jwtPayloadData";
import { AuthMiddleware } from "../middleware/global/AuthMiddleware";
import { ActivateCustomerValidator } from "../middleware/validator/customer/ActivateCustomerValidator";
import { CreateCustomerValidator } from "../middleware/validator/customer/CreateCustomerValidator";
import { GetCustomerPublicInfoByIdValidator } from "../middleware/validator/customer/GetCustomerPublicInfoByIdValidator";
import { UpdateCustomerValidator } from "../middleware/validator/customer/UpdateCustomerValidator";
import { CustomerService } from "../service/CustomerService";
import { JwtPayload } from "../service/dto/AuthServiceDto";
import { ActivateCustomerReqBodyDto, CreateCustomerReqBodyDto, CreateCustomerResponseDto, GetCustomerPublicInfoByIdRespBodyDto, GetRandomCustomerNameRespBodyDto, UpdateCustomerReqBodyDto } from "./dto/CustomerControllerDto";
@Service()
@JsonController('/customer')
export class CustomerControler {
public constructor(private customerService: CustomerService){}
@UseBefore(AuthMiddleware, UpdateCustomerValidator)
@Patch('/')
public async updateCustomer(
@JwtPayloadData() JwtPayloadData: JwtPayload,
@Body() body: UpdateCustomerReqBodyDto
):Promise<Customer> {
return await this.customerService.updateCustomerById(JwtPayloadData.customerId, body);
}
@UseBefore(GetCustomerPublicInfoByIdValidator)
@Get('/public/:customerId')
public async getCustomerPublicInfoById(
@Param('customerId') customerId: string
): Promise<GetCustomerPublicInfoByIdRespBodyDto> {
return await this.customerService.getCustomerPublicInfoById(customerId);
}
@Authorized()
@Get('/me')
public async getCustomerById(
@CurrentUser() customer: Customer | CustomerTemp
): Promise<Customer|CustomerTemp> {
return customer;
}
@UseBefore(CreateCustomerValidator)
@Post('/')
public async createCustomer(
@Body() body: CreateCustomerReqBodyDto
): Promise<CreateCustomerResponseDto> {
return await this.customerService.createCustomer(body);
}
@UseBefore(AuthMiddleware,ActivateCustomerValidator)
@Post('/activate')
public async activateCustomer(
@JwtPayloadData() jwtPayloadData: JwtPayload,
@Body() body: ActivateCustomerReqBodyDto
): Promise<Customer> {
return await this.customerService.activateCustomerById(jwtPayloadData.customerId, body);
}
@Get('/generate-name')
public async generateRandomCustomerName(
): Promise<GetRandomCustomerNameRespBodyDto> {
return await this.customerService.getRandomCustomerName();
}
}
================================================
FILE: aquila/search/src/controller/HomeController.ts
================================================
import { Get, JsonController } from "routing-controllers";
import { Service } from "typedi";
import { Queue } from 'bullmq';
@Service()
@JsonController('/')
export class HomeController {
@Get('/')
public index() {
const queue = new Queue('Paint', { connection: {
host: "redis-17256.c17.us-east-1-4.ec2.cloud.redislabs.com",
port: 17256,
username: 'default',
password: 'AbcRVEZFVHYjRNPFxAwLcSdUCA4DuUll'
}});
queue.add('cars', { color: 'blue' });
return {
msg: "Welcome to Aquila X"
}
}
}
================================================
FILE: aquila/search/src/controller/dto/AuthControllerDto.ts
================================================
import { AccountStatus } from "../../service/dto/AuthServiceDto";
export interface LoginCustomerRequestBodyDto {
secretKey: string;
}
export interface LoginCustomerResponseBodyDto {
token: string;
}
================================================
FILE: aquila/search/src/controller/dto/BookmarkControllerDto.ts
================================================
export interface AddBookmarkReqBodyDto {
html?: string;
url: string;
collectionId: string;
}
export interface GetBookmarksByCollectionIdReqQueryParamsDto {
query?: string;
limit?: string;
page?: string;
}
export interface GetBookmarksByCollectionIdResBodyDto {
totalPages: number;
totalRecords: number;
currentPage: number;
limit: number;
bookmarks: {
id: string;
collectionId: string;
url: string;
title: string;
author: string;
coverImg: string;
summary: string
description: string;
}[]
}
export interface GetFeaturedBookmarksReqQueryParamsDto {
limit?: string;
page?: string;
}
export interface GetFeaturedBookmarksResBodyDto {
totalPages: number;
totalRecords: number;
currentPage: number;
limit: number;
bookmarks: {
id: string;
collectionId: string;
url: string;
title: string;
author: string;
coverImg: string;
summary: string
description: string;
}[]
}
================================================
FILE: aquila/search/src/controller/dto/CollectionControllerDto.ts
================================================
import { Collection } from "../../entity/Collection";
import { CollectionTemp } from "../../entity/CollectionTemp";
export interface GetAllPublicCollectionReqQueryParamsDto {
limit?: string;
page?: string;
}
export interface GetAllPublicCollectionResPayloadDto {
totalRecords: number;
totalPages: number;
currentPage: number,
limit: number;
collections: Collection[] | CollectionTemp[]
}
export interface GetAllFeaturedCollectionReqQueryParamsDto {
limit?: string;
page?: string;
}
export interface GetAllFeaturedCollectionResPayloadDto {
totalRecords: number;
totalPages: number;
currentPage: number,
limit: number;
collections: Collection[] | CollectionTemp[]
}
================================================
FILE: aquila/search/src/controller/dto/CollectionSubscriptionControllerDto.ts
================================================
export interface GetSubscriptionsByCustomerIdReqQueryParamsDto {
query?: string;
limit?: string;
page?: string;
}
export interface GetSubscriptionsByCustomerIdResBodyDto {
totalPages: number;
totalRecords: number;
currentPage: number;
limit: number;
bookmarks: {
id: string;
collectionId: string;
url: string;
title: string;
author: string;
coverImg: string;
summary
gitextract_6lg6_8cc/
├── README.md
└── aquila/
├── encoder/
│ ├── CODE_OF_CONDUCT.md
│ ├── Dockerfile
│ ├── LICENSE
│ ├── README.md
│ ├── authentication.py
│ ├── config.yml
│ ├── encoder.py
│ ├── index.py
│ ├── ipfs.service
│ ├── manager.py
│ ├── requirements.txt
│ ├── run_tests.sh
│ ├── test/
│ │ ├── __init__.py
│ │ └── apis/
│ │ └── hub_fns.py
│ ├── test.py
│ └── utils/
│ ├── CID.py
│ ├── config.py
│ ├── cryptops.py
│ ├── downloader.py
│ └── schema.py
├── metricstore/
│ ├── .travis.yml
│ ├── CODE_OF_CONDUCT.md
│ ├── DB_config.yml
│ ├── Dockerfile
│ ├── DockerfileBig
│ ├── LICENSE
│ ├── README.md
│ ├── authentication.py
│ ├── index.py
│ ├── install.sh
│ ├── manager.py
│ ├── router.py
│ ├── run_tests.sh
│ ├── test/
│ │ ├── __init__.py
│ │ └── apis/
│ │ ├── auth_fns.py
│ │ ├── db_fns.py
│ │ ├── doc_fns.py
│ │ └── search_fns.py
│ ├── utils/
│ │ ├── CID.py
│ │ ├── config.py
│ │ ├── cryptops.py
│ │ └── schema.py
│ ├── vec_index/
│ │ ├── hannoy.py
│ │ └── hfaiss.py
│ └── wal/
│ ├── walman.py
│ └── walmon.py
├── network/
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── search/
│ ├── .dockerignore
│ ├── .nvmrc
│ ├── Dockerfile
│ ├── README.md
│ ├── package.json
│ ├── src/
│ │ ├── @types/
│ │ │ ├── express/
│ │ │ │ └── index.d.ts
│ │ │ └── index.d.ts
│ │ ├── app.ts
│ │ ├── config/
│ │ │ ├── db.ts
│ │ │ └── redisConnection.ts
│ │ ├── controller/
│ │ │ ├── AuthController.ts
│ │ │ ├── BookmarkController.ts
│ │ │ ├── CollectionController.ts
│ │ │ ├── CollectionSubscriptionController.ts
│ │ │ ├── CustomerController.ts
│ │ │ ├── HomeController.ts
│ │ │ └── dto/
│ │ │ ├── AuthControllerDto.ts
│ │ │ ├── BookmarkControllerDto.ts
│ │ │ ├── CollectionControllerDto.ts
│ │ │ ├── CollectionSubscriptionControllerDto.ts
│ │ │ └── CustomerControllerDto.ts
│ │ ├── entity/
│ │ │ ├── Bookmark.ts
│ │ │ ├── BookmarkPara.ts
│ │ │ ├── BookmarkParaTemp.ts
│ │ │ ├── BookmarkTemp.ts
│ │ │ ├── Collection.ts
│ │ │ ├── CollectionSubscription.ts
│ │ │ ├── CollectionSubscriptionTemp.ts
│ │ │ ├── CollectionTemp.ts
│ │ │ ├── Customer.ts
│ │ │ └── CustomerTemp.ts
│ │ ├── helper/
│ │ │ ├── auth.ts
│ │ │ ├── decorators/
│ │ │ │ └── jwtPayloadData.ts
│ │ │ └── user.ts
│ │ ├── index.ts
│ │ ├── job/
│ │ │ ├── AppQueue.ts
│ │ │ ├── appWorker.ts
│ │ │ ├── appWorkerProcessor.ts
│ │ │ └── types.ts
│ │ ├── lib/
│ │ │ ├── AquilaClientService.ts
│ │ │ └── ConfigService.ts
│ │ ├── middleware/
│ │ │ ├── global/
│ │ │ │ ├── AuthMiddleware.ts
│ │ │ │ ├── GlobalErrorMiddleware.ts
│ │ │ │ └── TokenParserMiddleware.ts
│ │ │ └── validator/
│ │ │ ├── bookmark/
│ │ │ │ ├── AddBookmarkValidator.ts
│ │ │ │ ├── GetBookmarkByCollectionIdValidator.ts
│ │ │ │ ├── GetFeaturedBookmarkValidator.ts
│ │ │ │ ├── GetPublicBookmarkByCollectionIdParamValidator.ts
│ │ │ │ └── GetPublicBookmarkByCollectionIdValidator.ts
│ │ │ ├── collection/
│ │ │ │ ├── GetAllFeaturedCollectionValidator.ts
│ │ │ │ ├── GetAllPublicCollectionValidator.ts
│ │ │ │ └── GetPublicCollectionByIdValidator.ts
│ │ │ ├── csubscription/
│ │ │ │ ├── SubscribeCollectionValidator.ts
│ │ │ │ └── UnSubscribeCollectionValidator.ts
│ │ │ └── customer/
│ │ │ ├── ActivateCustomerValidator.ts
│ │ │ ├── CreateCustomerValidator.ts
│ │ │ ├── GetCustomerPublicInfoByIdValidator.ts
│ │ │ └── UpdateCustomerValidator.ts
│ │ ├── migrations/
│ │ │ └── 1676714378990-bookmark.ts
│ │ ├── service/
│ │ │ ├── AuthService.ts
│ │ │ ├── BookmarkService.ts
│ │ │ ├── CollectionService.ts
│ │ │ ├── CollectionSubscriptionService.ts
│ │ │ ├── CustomerService.ts
│ │ │ └── dto/
│ │ │ ├── AuthServiceDto.ts
│ │ │ ├── BookmarkServiceDto.ts
│ │ │ ├── CollectionServiceDto.ts
│ │ │ ├── CollectionSubscriptionServiceDto.ts
│ │ │ └── CustomerServiceDto.ts
│ │ └── utils/
│ │ ├── errors/
│ │ │ └── ValidationError.ts
│ │ ├── randomAnimals.ts
│ │ └── validate.ts
│ └── tsconfig.json
├── txt_transform/
│ ├── Dockerfile_mercury
│ ├── Dockerfile_txtpick
│ ├── app.py
│ ├── package.json
│ ├── requirements.txt
│ ├── server.js
│ ├── test.html
│ └── test.py
└── view/
├── .dockerignore
├── .eslintrc.json
├── @types/
│ └── next-auth.d.ts
├── Dockerfile
├── README.md
├── components/
│ ├── hoc/
│ │ └── InitComponent.tsx
│ ├── layout/
│ │ ├── base/
│ │ │ └── baseLayout.tsx
│ │ ├── boxCenter/
│ │ │ ├── BoxCenterLayout.module.scss
│ │ │ └── BoxCenterLayout.tsx
│ │ ├── childLayout/
│ │ │ └── settings/
│ │ │ ├── Header.module.scss
│ │ │ ├── Header.tsx
│ │ │ ├── SettingsLayout.module.scss
│ │ │ ├── SettingsLayout.tsx
│ │ │ ├── Sidebar.module.scss
│ │ │ └── Sidebar.tsx
│ │ └── main/
│ │ ├── AddLink.module.scss
│ │ ├── AddLink.tsx
│ │ ├── Footer.module.scss
│ │ ├── Footer.tsx
│ │ ├── Header.module.scss
│ │ ├── Header.tsx
│ │ ├── MainLayout.module.scss
│ │ └── MainLayout.tsx
│ ├── pages/
│ │ ├── account/
│ │ │ └── editProfile/
│ │ │ ├── EditProfileForm.module.scss
│ │ │ ├── EditProfileForm.tsx
│ │ │ ├── EditProfileHeader.module.scss
│ │ │ ├── EditProfileHeader.tsx
│ │ │ ├── EditProfileWrapper.module.scss
│ │ │ └── EditProfileWrapper.tsx
│ │ ├── collection/
│ │ │ └── bookmark/
│ │ │ └── CollectionBookmarks/
│ │ │ ├── CollectionBookmarksPageWrapper.module.scss
│ │ │ ├── CollectionBookmarksPageWrapper.tsx
│ │ │ ├── SearchBar.module.scss
│ │ │ ├── SearchBar.tsx
│ │ │ ├── SearchPageProfile.module.scss
│ │ │ ├── SearchPageProfile.tsx
│ │ │ ├── SearchResultItem.module.scss
│ │ │ ├── SearchResultItem.tsx
│ │ │ ├── SearchResults.module.scss
│ │ │ └── SearchResults.tsx
│ │ ├── explore/
│ │ │ ├── Explore.module.scss
│ │ │ ├── Explore.tsx
│ │ │ ├── ExploreCategoryItem.module.scss
│ │ │ ├── ExploreCategoryItem.tsx
│ │ │ ├── ExploreCategoryList.module.scss
│ │ │ ├── ExploreCategoryList.tsx
│ │ │ └── ExplorePageWrapper.tsx
│ │ ├── home/
│ │ │ ├── HomePageWrapper.module.scss
│ │ │ ├── HomePageWrapper.tsx
│ │ │ ├── SearchBar.module.scss
│ │ │ ├── SearchBar.tsx
│ │ │ ├── SearchPageProfile.module.scss
│ │ │ ├── SearchPageProfile.tsx
│ │ │ ├── SearchResultItem.module.scss
│ │ │ ├── SearchResultItem.tsx
│ │ │ ├── SearchResults.module.scss
│ │ │ └── SearchResults.tsx
│ │ ├── index/
│ │ │ ├── AllControl.module.scss
│ │ │ ├── AllControl.tsx
│ │ │ ├── Discover.module.scss
│ │ │ ├── Discover.tsx
│ │ │ ├── Hero.module.scss
│ │ │ ├── Hero.tsx
│ │ │ ├── IndexPageWrapper.tsx
│ │ │ ├── Story.module.scss
│ │ │ └── Story.tsx
│ │ ├── signIn/
│ │ │ ├── SignInForm.module.scss
│ │ │ ├── SignInForm.tsx
│ │ │ └── SignInPageWrapper.tsx
│ │ ├── signUp/
│ │ │ ├── SecretKey.module.scss
│ │ │ ├── SecretKey.tsx
│ │ │ ├── SignUpForm.module.scss
│ │ │ ├── SignUpForm.tsx
│ │ │ └── SignUpPageWrapper.tsx
│ │ └── subscription/
│ │ ├── Collection.module.scss
│ │ ├── Collection.tsx
│ │ ├── HomePageWrapper.tsx
│ │ ├── SearchBar.module.scss
│ │ ├── SearchBar.tsx
│ │ ├── SearchPageProfile.module.scss
│ │ ├── SearchPageProfile.tsx
│ │ ├── SearchResultItem.module.scss
│ │ ├── SearchResultItem.tsx
│ │ ├── SearchResults.module.scss
│ │ ├── SearchResults.tsx
│ │ ├── SubscribedCollections.module.scss
│ │ ├── SubscribedCollections.tsx
│ │ ├── SubscriptionPageWrapper.module.scss
│ │ └── SubscriptionPageWrapper.tsx
│ └── ui/
│ ├── alert/
│ │ ├── Alert.module.scss
│ │ └── Alert.tsx
│ ├── layout/
│ │ ├── Container.module.scss
│ │ └── Container.tsx
│ ├── modal/
│ │ ├── Modal.module.scss
│ │ └── Modal.tsx
│ └── progressLoader/
│ ├── ProgressLoader.module.scss
│ └── ProgressLoader.tsx
├── middleware.ts
├── next.config.js
├── package.json
├── pages/
│ ├── _app.tsx
│ ├── _document.tsx
│ ├── account/
│ │ └── edit-profile.tsx
│ ├── api/
│ │ ├── auth/
│ │ │ └── [...nextauth].ts
│ │ └── hello.ts
│ ├── collection/
│ │ └── bookmark/
│ │ └── [collectionId].tsx
│ ├── explore.tsx
│ ├── home.tsx
│ ├── index.tsx
│ ├── sign-in.tsx
│ ├── sign-up.tsx
│ ├── subscription.tsx
│ └── test.tsx
├── store/
│ ├── index.ts
│ └── slices/
│ ├── auth.ts
│ ├── bookmark/
│ │ ├── addLink.ts
│ │ ├── getLoggedInCustBookmarksByCollectionId.ts
│ │ └── getPublicBookmarksByCollectionId.ts
│ ├── collection/
│ │ ├── getAllPublicCollections.ts
│ │ ├── getCollectionById.ts
│ │ ├── getCustomerSubscriptions.ts
│ │ ├── getFeaturedCollections.ts
│ │ ├── getLoggedInCustCollections.ts
│ │ ├── getSubscribedCollections.ts
│ │ ├── isCollectionSubscribed.ts
│ │ ├── subscribeCollectionById.ts
│ │ └── unSubscribeCollectionById.ts
│ ├── customer/
│ │ ├── activateCustomer.ts
│ │ ├── getCurrentLoggedInCustomer.ts
│ │ ├── getCustomerById.ts
│ │ └── updateCustomer.ts
│ ├── errors/
│ │ └── AsyncThunkSubmissionError.ts
│ ├── generateName.ts
│ ├── signup.ts
│ ├── types/
│ │ ├── Bookmark.ts
│ │ ├── Collection.ts
│ │ ├── CollectionSubscription.ts
│ │ ├── Customer.ts
│ │ └── validationErrors.ts
│ └── utils/
│ └── createError.ts
├── styles/
│ └── globals.scss
├── tsconfig.json
└── utils/
└── api.ts
SYMBOL INDEX (510 symbols across 163 files)
FILE: aquila/encoder/authentication.py
function check (line 7) | def check (json_data, signature):
FILE: aquila/encoder/encoder.py
function get_url (line 23) | def get_url (schema):
function get_url_hash (line 33) | def get_url_hash (url):
function download_model (line 38) | def download_model (url, directory, file_name):
function memload_model (line 59) | def memload_model (model_type, model_filename):
class EncodeRequest (line 79) | class EncodeRequest ():
method __init__ (line 80) | def __init__(self, id_in, text_in):
class Encoder (line 84) | class Encoder ():
method __init__ (line 85) | def __init__(self, encoder_name_in, request_queue_in):
method __del__ (line 94) | def __del__(self):
method count_request_id (line 97) | def count_request_id (self):
method preload_model (line 102) | def preload_model (self, json_schema, database_name):
method enqueue_compress_data (line 141) | async def enqueue_compress_data (self, texts):
method process_queue (line 151) | async def process_queue (self):
FILE: aquila/encoder/index.py
function quartserver (line 16) | def quartserver ():
function authenticate (line 23) | def authenticate ():
function extract_request_params (line 40) | async def extract_request_params (request):
function info (line 53) | def info ():
function prepare_model (line 66) | async def prepare_model ():
function compress_data (line 102) | async def compress_data ():
function init_variables (line 138) | async def init_variables():
function init_tasks (line 150) | async def init_tasks():
function shutdown (line 156) | async def shutdown():
FILE: aquila/encoder/manager.py
function get_database_name (line 16) | def get_database_name (schema_in):
function get_encoder_name (line 30) | def get_encoder_name (schema_in):
class Manager (line 44) | class Manager ():
method __init__ (line 45) | def __init__ (self):
method __del__ (line 50) | def __del__ (self):
method persist_db (line 53) | def persist_db (self, json_schema, database_name, persist=True):
method prepare_hub (line 82) | def prepare_hub (self):
method preload_model (line 96) | def preload_model (self, json_schema, persist=True):
method compress_data (line 132) | async def compress_data (self, database_name, texts):
method background_task (line 159) | async def background_task(self):
FILE: aquila/encoder/test.py
function run_test (line 27) | def run_test (index_in):
function test_concurrency_thread (line 42) | def test_concurrency_thread (counter_upto):
FILE: aquila/encoder/test/apis/hub_fns.py
class TestAuth (line 23) | class TestAuth (unittest.TestCase):
method test_1_auth_preload_http (line 26) | def test_1_auth_preload_http (self):
method test_2_auth_preload_ipfs (line 92) | def test_2_auth_preload_ipfs (self):
FILE: aquila/encoder/utils/CID.py
function doc2CID (line 7) | def doc2CID (inp):
function doc2bson (line 22) | def doc2bson (inp):
function bson2doc (line 32) | def bson2doc (inp):
FILE: aquila/encoder/utils/cryptops.py
function read_public_key (line 13) | def read_public_key (location):
function verify_signature (line 21) | def verify_signature (json_data, pub_key, signature):
FILE: aquila/encoder/utils/downloader.py
function download_large_file (line 9) | def download_large_file (url, location, method="get"):
function http_download (line 34) | def http_download (url, directory, file_name):
function ipfs_download (line 63) | def ipfs_download (url, directory, file_name):
FILE: aquila/encoder/utils/schema.py
function generate_schema (line 5) | def generate_schema (template):
function compile (line 89) | def compile (schema_def):
function validate_json_docs (line 95) | def validate_json_docs (validator, json_doc):
FILE: aquila/metricstore/authentication.py
function check (line 7) | def check (json_data, signature):
FILE: aquila/metricstore/index.py
function authenticate (line 22) | def authenticate ():
function extract_request_params (line 39) | def extract_request_params (request):
function info (line 51) | def info ():
function db_create (line 64) | def db_create ():
function doc_insert (line 96) | def doc_insert ():
function doc_delete (line 122) | def doc_delete ():
function db_search (line 148) | def db_search ():
function flaskserver (line 174) | def flaskserver ():
FILE: aquila/metricstore/manager.py
function byt (line 30) | def byt (inp):
class VecManager (line 33) | class VecManager:
method __init__ (line 35) | def __init__ (self, json_schema):
method __del__ (line 74) | def __del__(self):
method add_vectors (line 82) | def add_vectors (self, documents):
method delete_vectors (line 118) | def delete_vectors (self, cids):
method get_nearest (line 137) | def get_nearest (self, qmatrix, k, rad):
method get_index (line 165) | def get_index (self, location):
method resize_vector (line 189) | def resize_vector (self, vector, dim):
method resize_matrix (line 200) | def resize_matrix (self, matrix_, dim):
method swap_index (line 221) | def swap_index (self, location):
method update_training_data (line 247) | def update_training_data (self, documents):
method save_TD_to_disk (line 251) | def save_TD_to_disk (self):
method load_TD_from_disk (line 261) | def load_TD_from_disk (self):
method process (line 272) | def process (self):
method spawn (line 295) | def spawn (self):
FILE: aquila/metricstore/router.py
function preload_databases (line 17) | def preload_databases ():
function create_database (line 44) | def create_database (json_schema):
function load_database (line 84) | def load_database (database_name):
function insert_docs (line 91) | def insert_docs (docs, database_name):
function delete_docs (line 130) | def delete_docs (ids, database_name):
function search (line 143) | def search (matrix, k, rad, database_name):
FILE: aquila/metricstore/test/apis/auth_fns.py
class TestAuth (line 24) | class TestAuth (unittest.TestCase):
method test_1_auth_create_db (line 27) | def test_1_auth_create_db (self):
FILE: aquila/metricstore/test/apis/db_fns.py
class TestDB (line 7) | class TestDB (unittest.TestCase):
method test_1_db_fresh_create (line 10) | def test_1_db_fresh_create (self):
method test_2_db_exist_create (line 39) | def test_2_db_exist_create (self):
FILE: aquila/metricstore/test/apis/doc_fns.py
class TestDocs (line 10) | class TestDocs (unittest.TestCase):
method test_1_doc_fresh_create (line 13) | def test_1_doc_fresh_create (self):
method test_1a_doc_incomplete_create (line 46) | def test_1a_doc_incomplete_create (self):
method test_2_doc_exist_create (line 77) | def test_2_doc_exist_create (self):
method test_3_db_fresh_doc_create (line 110) | def test_3_db_fresh_doc_create (self):
method test_4_doc_fresh_delete (line 133) | def test_4_doc_fresh_delete (self):
method test_5a_doc_exist_delete_small (line 153) | def test_5a_doc_exist_delete_small (self):
method test_5b_doc_exist_delete_large (line 207) | def test_5b_doc_exist_delete_large (self):
method test_6_db_fresh_doc_delete (line 263) | def test_6_db_fresh_doc_delete (self):
FILE: aquila/metricstore/test/apis/search_fns.py
class TestSearch (line 11) | class TestSearch (unittest.TestCase):
method test_1_insert_and_search_small (line 14) | def test_1_insert_and_search_small (self):
method test_2_insert_and_search_large (line 44) | def test_2_insert_and_search_large (self):
FILE: aquila/metricstore/utils/CID.py
function doc2CID (line 7) | def doc2CID (inp):
function doc2bson (line 22) | def doc2bson (inp):
function bson2doc (line 32) | def bson2doc (inp):
FILE: aquila/metricstore/utils/cryptops.py
function read_public_key (line 13) | def read_public_key (location):
function verify_signature (line 21) | def verify_signature (json_data, pub_key, signature):
FILE: aquila/metricstore/utils/schema.py
function generate_schema (line 5) | def generate_schema (template):
function compile (line 90) | def compile (schema_def):
function validate_json_docs (line 96) | def validate_json_docs (validator, json_doc):
FILE: aquila/metricstore/vec_index/hannoy.py
class Annoy (line 12) | class Annoy:
method __init__ (line 13) | def __init__(self, model_location):
method __del__ (line 45) | def __del__(self):
method spawn (line 50) | def spawn (self):
method init_annoy (line 59) | def init_annoy(self):
method add_vectors (line 75) | def add_vectors(self, documents):
method process (line 81) | def process(self):
method delete_vectors (line 136) | def delete_vectors(self, ids):
method get_nearest_k (line 142) | def get_nearest_k(self, matrix, k):
method get_nearest_rad (line 158) | def get_nearest_rad(self, matrix, rad):
method load_model_from_disk (line 174) | def load_model_from_disk(self):
method save_model_to_disk (line 191) | def save_model_to_disk(self):
FILE: aquila/metricstore/vec_index/hfaiss.py
class Faiss (line 12) | class Faiss:
method __init__ (line 13) | def __init__(self, model_location):
method __del__ (line 39) | def __del__(self):
method spawn (line 44) | def spawn (self):
method init_faiss (line 53) | def init_faiss(self, matrix):
method is_initiated (line 71) | def is_initiated(self):
method load_model_from_disk (line 74) | def load_model_from_disk(self, location):
method save_model_to_disk (line 84) | def save_model_to_disk(self, location, index):
method add_vectors (line 94) | def add_vectors(self, documents):
method process (line 100) | def process(self):
method delete_vectors (line 147) | def delete_vectors(self, ids):
method get_nearest_rad (line 153) | def get_nearest_rad(self, matrix, rad):
method get_nearest_k (line 163) | def get_nearest_k(self, matrix, k):
FILE: aquila/metricstore/wal/walman.py
function create_new_chunk (line 32) | def create_new_chunk ():
function spawn (line 47) | def spawn ():
function add_log (line 60) | def add_log (batch_in):
function terminate (line 65) | def terminate ():
function process (line 69) | def process ():
FILE: aquila/network/main.go
type Document (line 30) | type Document struct
function homePage (line 38) | func homePage(w http.ResponseWriter, r *http.Request) {
function createNewDocument (line 43) | func createNewDocument(w http.ResponseWriter, r *http.Request) {
function deleteDocument (line 84) | func deleteDocument(w http.ResponseWriter, r *http.Request) {
function getDocuments (line 120) | func getDocuments(selector string) []Document {
function getVersion (line 143) | func getVersion(document Document) []byte {
function authenticate (line 156) | func authenticate() (int, []byte) {
function checkDB (line 161) | func checkDB(dbName string) (int, []byte) {
function createDB (line 165) | func createDB(dbName string) (int, []byte) {
function getDBInfo (line 169) | func getDBInfo(dbName string) (int, []byte) {
function getReplicationLog (line 173) | func getReplicationLog(dbName string, logID string) (int, []byte) {
function addBatchDocs (line 177) | func addBatchDocs(dbName string, documents []Document) (int, []byte) {
function getChanges (line 187) | func getChanges(dbName string) (int, []byte) {
function ensureCommit (line 191) | func ensureCommit(dbName string) (int, []byte) {
function setReplChkPoint (line 195) | func setReplChkPoint(dbName string, replLog []byte) (int, []byte) {
function request (line 199) | func request(url string, method string, payload string, contentType stri...
function handleRequests (line 239) | func handleRequests() {
function replicatorDemon (line 247) | func replicatorDemon() {
function main (line 353) | func main() {
FILE: aquila/search/src/@types/express/index.d.ts
type Request (line 5) | interface Request {
FILE: aquila/search/src/@types/index.d.ts
type ExtractorArgs (line 2) | type ExtractorArgs = any;
type ParserOptions (line 4) | type ParserOptions = {
type ParserResult (line 14) | type ParserResult = {
FILE: aquila/search/src/app.ts
function main (line 18) | async function main() {
FILE: aquila/search/src/controller/AuthController.ts
class AuthController (line 8) | class AuthController {
method constructor (line 10) | public constructor(private authService: AuthService) {}
method login (line 13) | public async login(
FILE: aquila/search/src/controller/BookmarkController.ts
class BookmarkController (line 20) | class BookmarkController {
method constructor (line 22) | public constructor(private bookmarkService: BookmarkService) {}
method addBookmark (line 26) | public async addBookmark(
method getBookmarksByCollectionId (line 35) | public async getBookmarksByCollectionId(
method getPublicBookmarksByCollectionId (line 52) | public async getPublicBookmarksByCollectionId(
method getAllFeaturedBookmark (line 68) | public async getAllFeaturedBookmark(
FILE: aquila/search/src/controller/CollectionController.ts
class CollectionController (line 16) | class CollectionController {
method constructor (line 18) | public constructor(private collectionService: CollectionService) {}
method getAllPublicCollection (line 22) | public async getAllPublicCollection(
method getAllPublicFeaturedCollection (line 37) | public async getAllPublicFeaturedCollection(
method getPublicCollectionById (line 53) | public async getPublicCollectionById(
method getCurrentCustomerCollections (line 61) | public async getCurrentCustomerCollections(
FILE: aquila/search/src/controller/CollectionSubscriptionController.ts
class CollectionSubscriptionController (line 18) | class CollectionSubscriptionController {
method constructor (line 20) | public constructor(private collectionSubscriptionService: CollectionSu...
method getCurrentCustomerSubscriptions (line 24) | public async getCurrentCustomerSubscriptions(
method subscribeCollection (line 33) | public async subscribeCollection(
method isCollectionAlreadySubscribed (line 43) | public async isCollectionAlreadySubscribed(
method unSubscribeCollection (line 53) | public async unSubscribeCollection(
method getSubscriptionsByCustomerId (line 63) | public async getSubscriptionsByCustomerId(
method getCustomerSubscribedCollections (line 79) | public async getCustomerSubscribedCollections(
FILE: aquila/search/src/controller/CustomerController.ts
class CustomerControler (line 18) | class CustomerControler {
method constructor (line 20) | public constructor(private customerService: CustomerService){}
method updateCustomer (line 24) | public async updateCustomer(
method getCustomerPublicInfoById (line 33) | public async getCustomerPublicInfoById(
method getCustomerById (line 41) | public async getCustomerById(
method createCustomer (line 49) | public async createCustomer(
method activateCustomer (line 57) | public async activateCustomer(
method generateRandomCustomerName (line 65) | public async generateRandomCustomerName(
FILE: aquila/search/src/controller/HomeController.ts
class HomeController (line 7) | class HomeController {
method index (line 10) | public index() {
FILE: aquila/search/src/controller/dto/AuthControllerDto.ts
type LoginCustomerRequestBodyDto (line 3) | interface LoginCustomerRequestBodyDto {
type LoginCustomerResponseBodyDto (line 7) | interface LoginCustomerResponseBodyDto {
FILE: aquila/search/src/controller/dto/BookmarkControllerDto.ts
type AddBookmarkReqBodyDto (line 1) | interface AddBookmarkReqBodyDto {
type GetBookmarksByCollectionIdReqQueryParamsDto (line 7) | interface GetBookmarksByCollectionIdReqQueryParamsDto {
type GetBookmarksByCollectionIdResBodyDto (line 13) | interface GetBookmarksByCollectionIdResBodyDto {
type GetFeaturedBookmarksReqQueryParamsDto (line 30) | interface GetFeaturedBookmarksReqQueryParamsDto {
type GetFeaturedBookmarksResBodyDto (line 35) | interface GetFeaturedBookmarksResBodyDto {
FILE: aquila/search/src/controller/dto/CollectionControllerDto.ts
type GetAllPublicCollectionReqQueryParamsDto (line 4) | interface GetAllPublicCollectionReqQueryParamsDto {
type GetAllPublicCollectionResPayloadDto (line 9) | interface GetAllPublicCollectionResPayloadDto {
type GetAllFeaturedCollectionReqQueryParamsDto (line 17) | interface GetAllFeaturedCollectionReqQueryParamsDto {
type GetAllFeaturedCollectionResPayloadDto (line 22) | interface GetAllFeaturedCollectionResPayloadDto {
FILE: aquila/search/src/controller/dto/CollectionSubscriptionControllerDto.ts
type GetSubscriptionsByCustomerIdReqQueryParamsDto (line 1) | interface GetSubscriptionsByCustomerIdReqQueryParamsDto {
type GetSubscriptionsByCustomerIdResBodyDto (line 7) | interface GetSubscriptionsByCustomerIdResBodyDto {
FILE: aquila/search/src/controller/dto/CustomerControllerDto.ts
type CreateCustomerReqBodyDto (line 4) | interface CreateCustomerReqBodyDto {
type CreateCustomerResponseDto (line 9) | interface CreateCustomerResponseDto {
type ActivateCustomerReqBodyDto (line 14) | interface ActivateCustomerReqBodyDto {
type UpdateCustomerReqBodyDto (line 22) | interface UpdateCustomerReqBodyDto {
type GetCustomerPublicInfoByIdRespBodyDto (line 30) | interface GetCustomerPublicInfoByIdRespBodyDto {
type GetRandomCustomerNameRespBodyDto (line 38) | interface GetRandomCustomerNameRespBodyDto {
FILE: aquila/search/src/entity/Bookmark.ts
type BookmarkStatus (line 3) | enum BookmarkStatus {
class Bookmark (line 10) | class Bookmark extends BaseEntity {
FILE: aquila/search/src/entity/BookmarkPara.ts
class BookmarkPara (line 4) | class BookmarkPara extends BaseEntity {
FILE: aquila/search/src/entity/BookmarkParaTemp.ts
class BookmarkParaTemp (line 4) | class BookmarkParaTemp extends BaseEntity {
FILE: aquila/search/src/entity/BookmarkTemp.ts
type BookmarkTempStatus (line 3) | enum BookmarkTempStatus {
class BookmarkTemp (line 10) | class BookmarkTemp extends BaseEntity {
FILE: aquila/search/src/entity/Collection.ts
class Collection (line 5) | class Collection extends BaseEntity {
FILE: aquila/search/src/entity/CollectionSubscription.ts
class CollectionSubscription (line 4) | class CollectionSubscription extends BaseEntity {
FILE: aquila/search/src/entity/CollectionSubscriptionTemp.ts
class CollectionSubscriptionTemp (line 4) | class CollectionSubscriptionTemp extends BaseEntity {
FILE: aquila/search/src/entity/CollectionTemp.ts
class CollectionTemp (line 4) | class CollectionTemp extends BaseEntity {
FILE: aquila/search/src/entity/Customer.ts
class Customer (line 4) | class Customer extends BaseEntity {
FILE: aquila/search/src/entity/CustomerTemp.ts
class CustomerTemp (line 4) | class CustomerTemp extends BaseEntity {
FILE: aquila/search/src/helper/auth.ts
function authorizationChecker (line 5) | function authorizationChecker(action: Action): boolean {
FILE: aquila/search/src/helper/decorators/jwtPayloadData.ts
function JwtPayloadData (line 5) | function JwtPayloadData() {
FILE: aquila/search/src/helper/user.ts
function currentUserChecker (line 8) | async function currentUserChecker(action: Action): Promise<Customer|Cust...
FILE: aquila/search/src/job/AppQueue.ts
class AppQueue (line 8) | class AppQueue {
method constructor (line 11) | public constructor(){
method add (line 15) | public async add<T>(name: AppJobNames, data: T) {
FILE: aquila/search/src/job/appWorker.ts
function getAppWorker (line 4) | function getAppWorker(connection: Redis) {
FILE: aquila/search/src/job/appWorkerProcessor.ts
function summarize (line 20) | async function summarize(html: string) {
FILE: aquila/search/src/job/types.ts
type AppJobNames (line 5) | enum AppJobNames {
type IndexDocumentData (line 9) | interface IndexDocumentData {
type AddData (line 14) | interface AddData {
type AppJobData (line 18) | type AppJobData = IndexDocumentData;
FILE: aquila/search/src/lib/AquilaClientService.ts
class AquilaClientService (line 8) | class AquilaClientService {
method constructor (line 12) | public constructor(private configService: ConfigService) {}
method connect (line 14) | public async connect() {
method getDbServer (line 28) | public getDbServer() {
method getHubServer (line 32) | public getHubServer() {
method createCollection (line 36) | public async createCollection(desc: string, secretKey: string) {
FILE: aquila/search/src/lib/ConfigService.ts
class ConfigService (line 6) | class ConfigService {
method constructor (line 7) | constructor() {
method loadEnv (line 11) | private loadEnv() {
method get (line 32) | get(key: string, defaultValue?: string): string {
FILE: aquila/search/src/middleware/global/AuthMiddleware.ts
class AuthMiddleware (line 7) | class AuthMiddleware implements ExpressMiddlewareInterface {
method constructor (line 8) | public constructor(private authService: AuthService) {}
method use (line 10) | public use(req: Request, res: Response, next: NextFunction): void {
FILE: aquila/search/src/middleware/global/GlobalErrorMiddleware.ts
class GlobalErrorHandler (line 8) | class GlobalErrorHandler implements ExpressErrorMiddlewareInterface {
method error (line 9) | error(err: Error, req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/global/TokenParserMiddleware.ts
class TokenParserMiddleware (line 8) | class TokenParserMiddleware implements ExpressMiddlewareInterface {
method constructor (line 9) | public constructor(private authService: AuthService) {}
method use (line 11) | public use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/bookmark/AddBookmarkValidator.ts
class AddBookmarkValidator (line 12) | class AddBookmarkValidator implements ExpressMiddlewareInterface {
method constructor (line 13) | public constructor(private collectionService: CollectionService, @JwtP...
method use (line 16) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/bookmark/GetBookmarkByCollectionIdValidator.ts
class GetBookmarkByCollectionIdValidator (line 9) | class GetBookmarkByCollectionIdValidator implements ExpressMiddlewareInt...
method use (line 10) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/bookmark/GetFeaturedBookmarkValidator.ts
class GetFeaturedBookmarkValidator (line 8) | class GetFeaturedBookmarkValidator implements ExpressMiddlewareInterface {
method use (line 9) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/bookmark/GetPublicBookmarkByCollectionIdParamValidator.ts
class GetPublicBookmarkByCollectionIdParamValidator (line 10) | class GetPublicBookmarkByCollectionIdParamValidator implements ExpressMi...
method use (line 11) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/bookmark/GetPublicBookmarkByCollectionIdValidator.ts
class GetPublicBookmarkByCollectionIdValidator (line 10) | class GetPublicBookmarkByCollectionIdValidator implements ExpressMiddlew...
method use (line 11) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/collection/GetAllFeaturedCollectionValidator.ts
class GetAllFeaturedCollectionValidator (line 8) | class GetAllFeaturedCollectionValidator implements ExpressMiddlewareInte...
method use (line 9) | async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/collection/GetAllPublicCollectionValidator.ts
class GetAllPublicCollectionValidator (line 8) | class GetAllPublicCollectionValidator implements ExpressMiddlewareInterf...
method use (line 9) | async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/collection/GetPublicCollectionByIdValidator.ts
class GetPublicCollectionByIdValidator (line 8) | class GetPublicCollectionByIdValidator implements ExpressMiddlewareInter...
method use (line 9) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/csubscription/SubscribeCollectionValidator.ts
class SubscribeCollectionValidator (line 10) | class SubscribeCollectionValidator implements ExpressMiddlewareInterface {
method constructor (line 11) | public constructor(private collectionSubService: CollectionSubscriptio...
method use (line 14) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/csubscription/UnSubscribeCollectionValidator.ts
class UnSubscribeCollectionValidator (line 9) | class UnSubscribeCollectionValidator implements ExpressMiddlewareInterfa...
method use (line 11) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/customer/ActivateCustomerValidator.ts
class ActivateCustomerValidator (line 9) | class ActivateCustomerValidator implements ExpressMiddlewareInterface {
method use (line 10) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/customer/CreateCustomerValidator.ts
class CreateCustomerValidator (line 8) | class CreateCustomerValidator implements ExpressMiddlewareInterface {
method use (line 9) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/customer/GetCustomerPublicInfoByIdValidator.ts
class GetCustomerPublicInfoByIdValidator (line 8) | class GetCustomerPublicInfoByIdValidator implements ExpressMiddlewareInt...
method use (line 9) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/middleware/validator/customer/UpdateCustomerValidator.ts
class UpdateCustomerValidator (line 11) | class UpdateCustomerValidator implements ExpressMiddlewareInterface {
method use (line 12) | public async use(req: Request, res: Response, next: NextFunction) {
FILE: aquila/search/src/migrations/1676714378990-bookmark.ts
class bookmark1676714378990 (line 3) | class bookmark1676714378990 implements MigrationInterface {
method up (line 6) | public async up(queryRunner: QueryRunner): Promise<void> {
method down (line 22) | public async down(queryRunner: QueryRunner): Promise<void> {
FILE: aquila/search/src/service/AuthService.ts
class AuthService (line 11) | class AuthService {
method constructor (line 13) | public constructor(private configService: ConfigService) {}
method authenticate (line 15) | public authenticate(token: string) {
method decodeToken (line 24) | public decodeToken(token: string): JwtPayload {
method generateToken (line 29) | private generateToken(payload: JwtPayload): string {
method login (line 36) | public async login(secretKey: string): Promise<string> {
method loginPermanentCustoemr (line 47) | public async loginPermanentCustoemr(secretKey: string): Promise<string>{
method loginTemporaryCustomer (line 66) | public async loginTemporaryCustomer(secretKey: string): Promise<string> {
FILE: aquila/search/src/service/BookmarkService.ts
class BookmarkService (line 22) | class BookmarkService {
method constructor (line 24) | public constructor(private appQueue: AppQueue, private aquilaClientSer...
method fetchWebsiteContent (line 26) | private async fetchWebsiteContent(url: string): Promise<string> {
method addTemporaryBookmark (line 40) | private async addTemporaryBookmark(data: AddBookmarkInputDto): Promise...
method addPermanentBookmark (line 64) | private async addPermanentBookmark(data: AddBookmarkInputDto): Promise...
method addBookmark (line 88) | public async addBookmark(data: AddBookmarkInputDto, accountStatus: Acc...
method getAllTemporaryBookmarksByCollectionId (line 95) | private async getAllTemporaryBookmarksByCollectionId(collectionId: str...
method getAllPermanentBookmarksByCollectionId (line 123) | private async getAllPermanentBookmarksByCollectionId(collectionId: str...
method getTemporaryBookmarksByCollectionId (line 151) | private async getTemporaryBookmarksByCollectionId(collectionId: string...
method getPermanentBookmarksByCollectionId (line 217) | private async getPermanentBookmarksByCollectionId(collectionId: string...
method getBookmarksByCollectionId (line 283) | public async getBookmarksByCollectionId(collectionId: string, options:...
method getFeaturedBookmarks (line 290) | public async getFeaturedBookmarks(options: GetBookmarksByCollectionIdO...
FILE: aquila/search/src/service/CollectionService.ts
class CollectionService (line 10) | class CollectionService {
method constructor (line 12) | public constructor() {}
method getAllTemporaryCollections (line 14) | private async getAllTemporaryCollections(options: GetAllCollectionsInp...
method getAllPermanentCollections (line 39) | public async getAllPermanentCollections(options: GetAllCollectionsInpu...
method getAllCollections (line 63) | public async getAllCollections(options: GetAllCollectionsInputOptionsD...
method getTempCustomerCollectionsByCustomerId (line 70) | private async getTempCustomerCollectionsByCustomerId(customerId: strin...
method getPermanentCustomerCollectionsByCustomerId (line 75) | private async getPermanentCustomerCollectionsByCustomerId(customerId: ...
method getCustomerCollectionsByCustomerId (line 80) | public async getCustomerCollectionsByCustomerId(customerId: string, ac...
method getTemporaryCollectionById (line 87) | public async getTemporaryCollectionById(id: string): Promise<Collectio...
method getPermanentCollectionById (line 95) | public async getPermanentCollectionById(id: string): Promise<Collectio...
method getCollectionById (line 103) | public async getCollectionById(id: string, accountStatus: AccountStatu...
method getPublicCollectionById (line 110) | public async getPublicCollectionById(id: string): Promise<Collection> {
FILE: aquila/search/src/service/CollectionSubscriptionService.ts
class CollectionSubscriptionService (line 16) | class CollectionSubscriptionService {
method constructor (line 18) | public constructor(private aquilaClientService: AquilaClientService) {}
method subscribeTemporaryCollection (line 20) | private async subscribeTemporaryCollection(collectionId: string, custo...
method subscribePermanentCollection (line 30) | private async subscribePermanentCollection(collectionId: string, custo...
method subscribeCollection (line 40) | public async subscribeCollection(collectionId: string, customerId: str...
method getTemporaryCollectionSubscriptionList (line 48) | private async getTemporaryCollectionSubscriptionList(customerId: strin...
method getPermanentCollectionSubscriptionList (line 53) | private async getPermanentCollectionSubscriptionList(customerId: strin...
method getCustomerSubscriptions (line 58) | public async getCustomerSubscriptions(customerId: string, accountStatu...
method isCollectionSubscribedByTemporaryCustomer (line 66) | public async isCollectionSubscribedByTemporaryCustomer(collectionId: s...
method isCollectionSubscribedByPermanentCustomer (line 77) | public async isCollectionSubscribedByPermanentCustomer(collectionId: s...
method isCollectionSubscribedByCustomer (line 88) | public async isCollectionSubscribedByCustomer(collectionId: string, cu...
method unSubscribeTemporaryCollection (line 96) | private async unSubscribeTemporaryCollection(collectionId: string, cus...
method unSubscribePermanentCollection (line 105) | private async unSubscribePermanentCollection(collectionId: string, cus...
method unSubscribeCollection (line 114) | public async unSubscribeCollection(collectionId: string, customerId: s...
method getTemporaryCollectionFollowerCount (line 122) | private async getTemporaryCollectionFollowerCount(collectionId: string...
method getPermanentCollectionFollowerCount (line 127) | private async getPermanentCollectionFollowerCount(collectionId: string...
method getCollectionSubscriberCount (line 132) | public async getCollectionSubscriberCount(collectionId: string, accoun...
method getTotalCollectionSubscribedByTemporaryCustomer (line 140) | public async getTotalCollectionSubscribedByTemporaryCustomer(subscribe...
method getTotalCollectionSubscribedByPermanentCustomer (line 145) | public async getTotalCollectionSubscribedByPermanentCustomer(subscribe...
method getTotalCollectionSubscribedByCustomer (line 150) | public async getTotalCollectionSubscribedByCustomer(customerId: string...
method getAllSubscriptionsByCustomerId (line 158) | private async getAllSubscriptionsByCustomerId(collectionIds: string[],...
method getSubscriptionsByTemporaryCustomerId (line 186) | public async getSubscriptionsByTemporaryCustomerId(customerId: string,...
method getSubscriptionsByPermanentCustomerId (line 253) | public async getSubscriptionsByPermanentCustomerId(customerId: string,...
method getSubscriptionsByCustomerId (line 320) | public async getSubscriptionsByCustomerId(customerId: string, options:...
method getCustomerSubscribedCollections (line 327) | public async getCustomerSubscribedCollections(customerId: string, acco...
FILE: aquila/search/src/service/CustomerService.ts
class CustomerService (line 25) | class CustomerService {
method constructor (line 27) | public constructor(private aquilaClientService: AquilaClientService) {}
method getRandomCustomerName (line 29) | public async getRandomCustomerName(): Promise<GetRandomCustomerNameOut...
method getCustomerById (line 38) | public async getCustomerById(id: string, accountStatus: AccountStatus) {
method getPermanentCustomerById (line 45) | private async getPermanentCustomerById(id: string): Promise<Customer> {
method getTemporaryCustomerById (line 53) | private async getTemporaryCustomerById(id: string): Promise<CustomerTe...
method createCustomer (line 61) | public async createCustomer(data: CreateCustomerInputDataDto): Promise...
method updateCustomerById (line 109) | public async updateCustomerById(id: string, data: UpdateCustomerByIdIn...
method getCustomerPublicInfoById (line 120) | public async getCustomerPublicInfoById(id: string): Promise<GetCustome...
method activateCustomerById (line 131) | public async activateCustomerById(id: string, data: ActivateCustomerBy...
method findCustomerByEmailId (line 229) | public async findCustomerByEmailId(email: string) {
FILE: aquila/search/src/service/dto/AuthServiceDto.ts
type AccountStatus (line 1) | enum AccountStatus {
type JwtPayload (line 6) | interface JwtPayload {
FILE: aquila/search/src/service/dto/BookmarkServiceDto.ts
type AddBookmarkInputDto (line 4) | interface AddBookmarkInputDto {
type GetBookmarksByCollectionIdOptionsInputDto (line 10) | interface GetBookmarksByCollectionIdOptionsInputDto {
type GetAllBookmarksByCollectionIdOptionsInputDto (line 16) | type GetAllBookmarksByCollectionIdOptionsInputDto = Omit<GetBookmarksByC...
type GetFeaturedBookmarksOptionsInputDto (line 18) | interface GetFeaturedBookmarksOptionsInputDto {
type BookmarkData (line 23) | interface BookmarkData {
type GetBookmarksByCollectionIdOutputDto (line 34) | interface GetBookmarksByCollectionIdOutputDto {
type GetFeaturedBookmarksOutputDto (line 42) | interface GetFeaturedBookmarksOutputDto {
FILE: aquila/search/src/service/dto/CollectionServiceDto.ts
type GetAllCollectionsInputOptionsDto (line 4) | interface GetAllCollectionsInputOptionsDto {
type GetAllCollectionsOutputDto (line 13) | interface GetAllCollectionsOutputDto {
FILE: aquila/search/src/service/dto/CollectionSubscriptionServiceDto.ts
type GetSubscriptionsByCustomerIdOptionsInputDto (line 1) | interface GetSubscriptionsByCustomerIdOptionsInputDto {
type BookmarkData (line 7) | interface BookmarkData {
type GetSubscriptionsByCustomerIdOutputDto (line 18) | interface GetSubscriptionsByCustomerIdOutputDto {
FILE: aquila/search/src/service/dto/CustomerServiceDto.ts
type CreateCustomerInputDataDto (line 4) | interface CreateCustomerInputDataDto {
type CreateCustomerOutputDto (line 9) | interface CreateCustomerOutputDto {
type ActivateCustomerByIdInputDataDto (line 14) | interface ActivateCustomerByIdInputDataDto {
type UpdateCustomerByIdInputDataDto (line 22) | interface UpdateCustomerByIdInputDataDto {
type GetCustomerPublicInfoByIdOutputDto (line 30) | interface GetCustomerPublicInfoByIdOutputDto {
type GetRandomCustomerNameOutputDto (line 38) | interface GetRandomCustomerNameOutputDto {
FILE: aquila/search/src/utils/errors/ValidationError.ts
class ValidationError (line 3) | class ValidationError extends HttpError {
method constructor (line 4) | constructor(message: string, public errors: any ) {
FILE: aquila/search/src/utils/randomAnimals.ts
function randomAnimal (line 4) | function randomAnimal(): [string, string] {
FILE: aquila/txt_transform/app.py
function get_paragraphs (line 28) | def get_paragraphs(html_doc):
function extract_request_params (line 51) | def extract_request_params (request):
function process (line 64) | def process ():
function flaskserver (line 109) | def flaskserver ():
FILE: aquila/txt_transform/test.py
function process_html (line 4) | def process_html (html, url):
function trim_content (line 24) | def trim_content (html):
FILE: aquila/view/@types/next-auth.d.ts
type User (line 4) | interface User {
type Session (line 13) | interface Session {
type JWT (line 26) | interface JWT {
FILE: aquila/view/components/hoc/InitComponent.tsx
type InitCompoentProps (line 6) | interface InitCompoentProps {
FILE: aquila/view/components/layout/base/baseLayout.tsx
type BaseLayoutProps (line 6) | interface BaseLayoutProps {
FILE: aquila/view/components/layout/childLayout/settings/Header.tsx
type HeaderProps (line 7) | interface HeaderProps {
FILE: aquila/view/components/layout/childLayout/settings/SettingsLayout.tsx
type SettingsLayoutProps (line 9) | interface SettingsLayoutProps {
FILE: aquila/view/components/layout/main/AddLink.tsx
type AddLinkFromData (line 10) | interface AddLinkFromData {
type AddLinkProps (line 14) | interface AddLinkProps {
FILE: aquila/view/components/layout/main/Footer.tsx
type FooterProps (line 9) | interface FooterProps {
FILE: aquila/view/components/layout/main/Header.tsx
type HeaderProps (line 15) | interface HeaderProps {
FILE: aquila/view/components/layout/main/MainLayout.tsx
type MainLayoutProps (line 16) | interface MainLayoutProps {
FILE: aquila/view/components/pages/account/editProfile/EditProfileForm.tsx
type EditProfileFormProps (line 8) | interface EditProfileFormProps {
type EditProfileFormData (line 16) | interface EditProfileFormData {
FILE: aquila/view/components/pages/account/editProfile/EditProfileWrapper.tsx
type EditProfileWrapperProps (line 11) | interface EditProfileWrapperProps {
FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/CollectionBookmarksPageWrapper.tsx
type CollectionBookmarksPageWrapperProps (line 16) | interface CollectionBookmarksPageWrapperProps {
FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchBar.tsx
type SearchBarProps (line 5) | interface SearchBarProps {
FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchPageProfile.tsx
type SearchPageProfileProps (line 10) | interface SearchPageProfileProps {
FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResultItem.tsx
type SearchResultItemProps (line 5) | interface SearchResultItemProps {
FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResults.tsx
type SearchResultsProps (line 6) | interface SearchResultsProps {
FILE: aquila/view/components/pages/explore/Explore.tsx
type ExploreProps (line 8) | interface ExploreProps {
FILE: aquila/view/components/pages/explore/ExploreCategoryItem.tsx
type ExploreItemProps (line 7) | interface ExploreItemProps {
FILE: aquila/view/components/pages/explore/ExploreCategoryList.tsx
type ExploreCategoryListProps (line 7) | interface ExploreCategoryListProps {
FILE: aquila/view/components/pages/explore/ExplorePageWrapper.tsx
type ExplorePageWrapperProps (line 7) | interface ExplorePageWrapperProps {
FILE: aquila/view/components/pages/home/HomePageWrapper.tsx
type HomePageWrapperProps (line 15) | interface HomePageWrapperProps {
FILE: aquila/view/components/pages/home/SearchBar.tsx
type SearchBarProps (line 5) | interface SearchBarProps {
FILE: aquila/view/components/pages/home/SearchPageProfile.tsx
type SearchPageProfileProps (line 10) | interface SearchPageProfileProps {
FILE: aquila/view/components/pages/home/SearchResultItem.tsx
type SearchResultItemProps (line 6) | interface SearchResultItemProps {
FILE: aquila/view/components/pages/home/SearchResults.tsx
type SearchResultsProps (line 6) | interface SearchResultsProps {
FILE: aquila/view/components/pages/signIn/SignInPageWrapper.tsx
type SignInPageWrapperProps (line 5) | interface SignInPageWrapperProps {
FILE: aquila/view/components/pages/signUp/SecretKey.tsx
type SecretKeyProps (line 10) | interface SecretKeyProps {
FILE: aquila/view/components/pages/signUp/SignUpForm.tsx
type SignUpFormProps (line 9) | interface SignUpFormProps {
type FormData (line 18) | type FormData = {
FILE: aquila/view/components/pages/signUp/SignUpPageWrapper.tsx
type SignUpPageWrapperProps (line 8) | interface SignUpPageWrapperProps {
FILE: aquila/view/components/pages/subscription/Collection.tsx
type CollectionProps (line 6) | interface CollectionProps {
FILE: aquila/view/components/pages/subscription/HomePageWrapper.tsx
type HomePageWrapperProps (line 14) | interface HomePageWrapperProps {
FILE: aquila/view/components/pages/subscription/SearchBar.tsx
type SearchBarProps (line 5) | interface SearchBarProps {
FILE: aquila/view/components/pages/subscription/SearchPageProfile.tsx
type SearchPageProfileProps (line 10) | interface SearchPageProfileProps {
FILE: aquila/view/components/pages/subscription/SearchResultItem.tsx
type SearchResultItemProps (line 6) | interface SearchResultItemProps {
FILE: aquila/view/components/pages/subscription/SearchResults.tsx
type SearchResultsProps (line 6) | interface SearchResultsProps {
FILE: aquila/view/components/pages/subscription/SubscribedCollections.tsx
type SubscribedCollectionsProps (line 7) | interface SubscribedCollectionsProps {
FILE: aquila/view/components/pages/subscription/SubscriptionPageWrapper.tsx
type SubscriptionPageWrapperProps (line 14) | interface SubscriptionPageWrapperProps {
FILE: aquila/view/components/ui/alert/Alert.tsx
type AlertProps (line 4) | interface AlertProps {
FILE: aquila/view/components/ui/layout/Container.tsx
type ContainerProps (line 5) | interface ContainerProps {
FILE: aquila/view/components/ui/modal/Modal.tsx
type ModelProps (line 6) | interface ModelProps {
FILE: aquila/view/components/ui/progressLoader/ProgressLoader.tsx
type ProgressLoaderCtxValue (line 4) | interface ProgressLoaderCtxValue { status: boolean, setLoader: (status: ...
type ProgressLoaderProviderProps (line 16) | interface ProgressLoaderProviderProps {
FILE: aquila/view/pages/_app.tsx
type ApplicationProps (line 13) | interface ApplicationProps {
function MyApp (line 17) | function MyApp({ Component, pageProps: { session, ...rest} }: AppProps<A...
FILE: aquila/view/pages/_document.tsx
function Document (line 3) | function Document() {
FILE: aquila/view/pages/api/auth/[...nextauth].ts
type AuthPayload (line 7) | interface AuthPayload {
method authorize (line 20) | async authorize(credentials){
method session (line 49) | session({ session, token}) {
method jwt (line 70) | jwt({ token, user}) {
FILE: aquila/view/pages/api/hello.ts
type Data (line 4) | type Data = {
function handler (line 8) | function handler(
FILE: aquila/view/pages/explore.tsx
type ExplorePageProps (line 11) | interface ExplorePageProps {
FILE: aquila/view/store/index.ts
type AppStore (line 56) | type AppStore = ReturnType<typeof createStore>;
type AppState (line 58) | type AppState = ReturnType<AppStore['getState']>;
type AppDispatch (line 60) | type AppDispatch = typeof store.dispatch;
type AppThunk (line 62) | type AppThunk<ReturnType = void> = ThunkAction<ReturnType, AppState, unk...
FILE: aquila/view/store/slices/auth.ts
type AuthState (line 5) | interface AuthState {
type SignInPayloadAction (line 18) | interface SignInPayloadAction {
FILE: aquila/view/store/slices/bookmark/addLink.ts
type AddLinkData (line 12) | interface AddLinkData {
type AddLinkReqPayload (line 17) | type AddLinkReqPayload = AddLinkData;
type AddLinkResPayload (line 19) | type AddLinkResPayload = Bookmark;
type AddLinkValidationErrors (line 21) | type AddLinkValidationErrors = ValidationErrors<AddLinkReqPayload>;
type AddLinkState (line 23) | interface AddLinkState {
FILE: aquila/view/store/slices/bookmark/getLoggedInCustBookmarksByCollectionId.ts
type GetLoggedInCustBookmarksByCollectionIdState (line 8) | interface GetLoggedInCustBookmarksByCollectionIdState {
type GetLoggedInCustBookmarksByCollectionIdData (line 19) | interface GetLoggedInCustBookmarksByCollectionIdData {
type GetLoggedInCustBookmarksByCollectionIdResPayload (line 28) | type GetLoggedInCustBookmarksByCollectionIdResPayload = Omit<GetLoggedIn...
type GetLoggedInCustBookmarksByCollectionIdInputOptions (line 30) | interface GetLoggedInCustBookmarksByCollectionIdInputOptions {
FILE: aquila/view/store/slices/bookmark/getPublicBookmarksByCollectionId.ts
type GetPublicBookmarksByCollectionIdState (line 8) | interface GetPublicBookmarksByCollectionIdState {
type GetPublicBookmarksByCollectionIdData (line 19) | interface GetPublicBookmarksByCollectionIdData {
type GetPublicBookmarksByCollectionIdResPayload (line 28) | type GetPublicBookmarksByCollectionIdResPayload = Omit<GetPublicBookmark...
type GetPublicBookmarksByCollectionIdInputOptions (line 30) | interface GetPublicBookmarksByCollectionIdInputOptions {
FILE: aquila/view/store/slices/collection/getAllPublicCollections.ts
type GetAllPublicCollectionsState (line 9) | interface GetAllPublicCollectionsState {
type GetAllPublicCollectionsData (line 19) | interface GetAllPublicCollectionsData {
type GetAllPublicCollectionsResPayload (line 27) | type GetAllPublicCollectionsResPayload = GetAllPublicCollectionsData;
FILE: aquila/view/store/slices/collection/getCollectionById.ts
type GetCollectionByIdState (line 7) | interface GetCollectionByIdState {
FILE: aquila/view/store/slices/collection/getCustomerSubscriptions.ts
type GetCustomerSubscriptionsState (line 8) | interface GetCustomerSubscriptionsState {
type GetCustomerSubscriptionsData (line 19) | interface GetCustomerSubscriptionsData {
type GetCustomerSubscriptionsResPayload (line 28) | type GetCustomerSubscriptionsResPayload = Omit<GetCustomerSubscriptionsD...
type GetCustomerSubscriptionsInputOptions (line 30) | interface GetCustomerSubscriptionsInputOptions {
FILE: aquila/view/store/slices/collection/getFeaturedCollections.ts
type GetFeaturedCollectionsState (line 9) | interface GetFeaturedCollectionsState {
type GetFeaturedCollectionsData (line 19) | interface GetFeaturedCollectionsData {
type GetFeaturedCollectionsResPayload (line 27) | type GetFeaturedCollectionsResPayload = GetFeaturedCollectionsData;
FILE: aquila/view/store/slices/collection/getLoggedInCustCollections.ts
type GetLoggedInCustCollectionsState (line 8) | interface GetLoggedInCustCollectionsState {
FILE: aquila/view/store/slices/collection/getSubscribedCollections.ts
type GetSubscribedCollectionsState (line 8) | interface GetSubscribedCollectionsState {
FILE: aquila/view/store/slices/collection/isCollectionSubscribed.ts
type IsCollectionSubscribedResPayload (line 12) | interface IsCollectionSubscribedResPayload {
type IsCollectionSubscribedState (line 17) | interface IsCollectionSubscribedState {
FILE: aquila/view/store/slices/collection/subscribeCollectionById.ts
type SubscribeCollectionByIdReqPayload (line 13) | interface SubscribeCollectionByIdReqPayload {
type SubscribeCollectionByIdResPayload (line 17) | type SubscribeCollectionByIdResPayload = CollectionSubscription;
type SubscribeCollectionByIdValidationErrors (line 19) | type SubscribeCollectionByIdValidationErrors = ValidationErrors<Subscrib...
type SubscribeCollectionByIdState (line 21) | interface SubscribeCollectionByIdState {
FILE: aquila/view/store/slices/collection/unSubscribeCollectionById.ts
type UnSubscribeCollectionByIdResPayload (line 12) | type UnSubscribeCollectionByIdResPayload = CollectionSubscription;
type UnSubscribeCollectionByIdState (line 15) | interface UnSubscribeCollectionByIdState {
FILE: aquila/view/store/slices/customer/activateCustomer.ts
type ActivateCustomerData (line 12) | interface ActivateCustomerData {
type ActivateCustomerReqPayload (line 20) | type ActivateCustomerReqPayload = ActivateCustomerData;
type ActivateCustomerResPayload (line 22) | type ActivateCustomerResPayload = Customer;
type ActivateCustomerValidationErrors (line 24) | type ActivateCustomerValidationErrors = ValidationErrors<ActivateCustome...
type ActivateCustomerState (line 26) | interface ActivateCustomerState {
FILE: aquila/view/store/slices/customer/getCurrentLoggedInCustomer.ts
type GetCurrentLoggedInCustomerState (line 7) | interface GetCurrentLoggedInCustomerState {
FILE: aquila/view/store/slices/customer/getCustomerById.ts
type GetCustomerByIdState (line 7) | interface GetCustomerByIdState {
FILE: aquila/view/store/slices/customer/updateCustomer.ts
type UpdateCustomerData (line 12) | interface UpdateCustomerData {
type UpdateCustomerReqPayload (line 20) | type UpdateCustomerReqPayload = UpdateCustomerData;
type UpdateCustomerResPayload (line 22) | type UpdateCustomerResPayload = Customer;
type UpdateCustomerValidationErrors (line 24) | type UpdateCustomerValidationErrors = ValidationErrors<UpdateCustomerData>;
type UpdateCustomerState (line 26) | interface UpdateCustomerState {
FILE: aquila/view/store/slices/errors/AsyncThunkSubmissionError.ts
class AsyncThunkSubmissionError (line 1) | class AsyncThunkSubmissionError<T = null> extends Error {
method constructor (line 4) | constructor(message: string, validationErrors: T | null) {
FILE: aquila/view/store/slices/generateName.ts
type GeneratedNameState (line 10) | interface GeneratedNameState {
FILE: aquila/view/store/slices/signup.ts
type Collection (line 10) | interface Collection {
type SignUpData (line 22) | interface SignUpData {
type SignUpRequestPayload (line 27) | type SignUpRequestPayload = SignUpData;
type SignUpResponsePayload (line 29) | interface SignUpResponsePayload {
type SignUpValidationErrors (line 34) | type SignUpValidationErrors = ValidationErrors<SignUpRequestPayload>
type SignUpState (line 36) | interface SignUpState {
FILE: aquila/view/store/slices/types/Bookmark.ts
type BookmarkStatus (line 3) | enum BookmarkStatus {
type Bookmark (line 9) | interface Bookmark {
FILE: aquila/view/store/slices/types/Collection.ts
type Collection (line 3) | interface Collection {
FILE: aquila/view/store/slices/types/CollectionSubscription.ts
type CollectionSubscription (line 1) | interface CollectionSubscription {
FILE: aquila/view/store/slices/types/Customer.ts
type Customer (line 1) | interface Customer {
FILE: aquila/view/store/slices/types/validationErrors.ts
type ValidationErrorData (line 1) | interface ValidationErrorData {
type ResponsePayloadValidationErrors (line 8) | type ResponsePayloadValidationErrors<T> = Partial<Record<keyof T, Valida...
type ValidationErrors (line 10) | type ValidationErrors<T> = Partial<Record<keyof T, string>>
FILE: aquila/view/store/slices/utils/createError.ts
type SubmissionErrorResponse (line 62) | interface SubmissionErrorResponse<T> {
Condensed preview — 273 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,249K chars).
[
{
"path": "README.md",
"chars": 8481,
"preview": "<div align=\"center\">\n <a href=\"https://aquila.network\">\n <img\n src=\"https://user-images.githubusercontent.com/1"
},
{
"path": "aquila/encoder/CODE_OF_CONDUCT.md",
"chars": 3360,
"preview": "# AquilaDB Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming envir"
},
{
"path": "aquila/encoder/Dockerfile",
"chars": 1124,
"preview": "# start a new build stage\nFROM ubuntu:latest as builder\n\n# set work directory\nENV ROOT_DIR /home/root\nWORKDIR $ROOT_DIR\n"
},
{
"path": "aquila/encoder/LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "aquila/encoder/README.md",
"chars": 1288,
"preview": "<div align=\"center\">\n <a href=\"https://aquila.network\">\n <img\n src=\"https://user-images.githubusercontent.com/1"
},
{
"path": "aquila/encoder/authentication.py",
"chars": 205,
"preview": "from utils import cryptops\n\nimport os\n\npub_key = cryptops.read_public_key(os.environ[\"AUTH_KEY_FILE\"])\n\ndef check (json_"
},
{
"path": "aquila/encoder/config.yml",
"chars": 76,
"preview": "auth:\n pubkey: \"/ossl/public.pem\"\nipfs:\n gateway: \"http://127.0.0.1:8080\"\n"
},
{
"path": "aquila/encoder/encoder.py",
"chars": 7149,
"preview": "import logging\n\nimport fasttext\nfrom utils import downloader\nimport hashlib\nimport base58\nimport json\n\nfrom sentence_tra"
},
{
"path": "aquila/encoder/index.py",
"chars": 4077,
"preview": "import logging\n\nfrom quart import Quart\nfrom quart import request\n\nfrom functools import wraps\nimport asyncio\n\nfrom util"
},
{
"path": "aquila/encoder/ipfs.service",
"chars": 242,
"preview": "[Unit]\nDescription=IPFS daemon\nAfter=network.target\n\n[Service]\n### custom ipfs datastore location\n# Environment=IPFS_PAT"
},
{
"path": "aquila/encoder/manager.py",
"chars": 6366,
"preview": "import logging\nlogging.basicConfig()\nlogging.getLogger().setLevel(logging.DEBUG)\n\nimport asyncio\nimport os\nimport json\ni"
},
{
"path": "aquila/encoder/requirements.txt",
"chars": 872,
"preview": "aiofiles==0.8.0\nbase58==2.1.1\nblinker==1.4\nbson==0.5.10\ncertifi==2021.10.8\nchardet==4.0.0\ncharset-normalizer==2.0.10\ncli"
},
{
"path": "aquila/encoder/run_tests.sh",
"chars": 57,
"preview": "# rm -r /data/*\n\npython3 -m unittest test.apis.hub_fns -v"
},
{
"path": "aquila/encoder/test/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "aquila/encoder/test/apis/hub_fns.py",
"chars": 4539,
"preview": "import unittest\n\nfrom utils import CID, schema\n\nfrom Crypto.Hash import SHA384\nfrom Crypto.PublicKey import RSA\nfrom Cry"
},
{
"path": "aquila/encoder/test.py",
"chars": 3845,
"preview": "from aquilapy import Wallet, DB, Hub\nimport numpy as np\nimport time\n\nimport multiprocessing as mp\n\n# Create a wallet ins"
},
{
"path": "aquila/encoder/utils/CID.py",
"chars": 837,
"preview": "import logging\n\nimport bson\nimport hashlib\nimport base58\n\ndef doc2CID (inp):\n try:\n # JSON OBJ to BSON encode\n"
},
{
"path": "aquila/encoder/utils/config.py",
"chars": 378,
"preview": "import yaml\nimport os\n\nos.environ[\"DATA_STORE_LOCATION\"] = \"/data/\"\n\nwith open(\"config.yml\", \"r\") as stream:\n DB_conf"
},
{
"path": "aquila/encoder/utils/cryptops.py",
"chars": 787,
"preview": "import logging\n\nfrom Crypto.Hash import SHA384\nfrom Crypto.PublicKey import RSA\nfrom Crypto.Signature import pkcs1_15\n\ni"
},
{
"path": "aquila/encoder/utils/downloader.py",
"chars": 3044,
"preview": "import logging\n\nimport os\nimport time\n\nimport requests\nfrom tqdm import tqdm\n\ndef download_large_file (url, location, me"
},
{
"path": "aquila/encoder/utils/schema.py",
"chars": 2876,
"preview": "import logging\n\nimport fastjsonschema\n\ndef generate_schema (template):\n # get all keys from template schema\n metad"
},
{
"path": "aquila/metricstore/.travis.yml",
"chars": 366,
"preview": "language: node_js\n\nservices:\n - docker\n\nnode_js:\n - \"11\"\n\nbefore_install:\n - docker build -f Dockerfile_local_build -"
},
{
"path": "aquila/metricstore/CODE_OF_CONDUCT.md",
"chars": 3360,
"preview": "# AquilaDB Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming envir"
},
{
"path": "aquila/metricstore/DB_config.yml",
"chars": 675,
"preview": "docs:\n cswap: 10101 # minimum data required to start indexing\n vd: 784 # fixed vector dimension\nfaiss:\n init:\n nli"
},
{
"path": "aquila/metricstore/Dockerfile",
"chars": 650,
"preview": "# start a new build stage\nFROM ubuntu:latest as builder\n\nRUN ls\n\n# set work directory\nENV ROOT_DIR /home/root\nWORKDIR $R"
},
{
"path": "aquila/metricstore/DockerfileBig",
"chars": 1078,
"preview": "# start a new build stage\nFROM ubuntu:latest as builder\n\n# set work directory\nENV ROOT_DIR /home/root\nWORKDIR $ROOT_DIR\n"
},
{
"path": "aquila/metricstore/LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "aquila/metricstore/README.md",
"chars": 8481,
"preview": "<div align=\"center\">\n <a href=\"https://aquila.network\">\n <img\n src=\"https://user-images.githubusercontent.com/1"
},
{
"path": "aquila/metricstore/authentication.py",
"chars": 205,
"preview": "from utils import cryptops\n\nimport os\n\npub_key = cryptops.read_public_key(os.environ[\"AUTH_KEY_FILE\"])\n\ndef check (json_"
},
{
"path": "aquila/metricstore/index.py",
"chars": 4124,
"preview": "import logging\n\nfrom flask import Flask, request\nfrom flask_cors import CORS\nfrom flask import jsonify\nfrom functools im"
},
{
"path": "aquila/metricstore/install.sh",
"chars": 2995,
"preview": "#!/bin/bash -e\n\nexport DEBIAN_FRONTEND=noninteractive\napt update\n\nexport USER=$(whoami)\nexport ROOT_DIR=/home/$USER\n\nmkd"
},
{
"path": "aquila/metricstore/manager.py",
"chars": 9521,
"preview": "import logging\n\nfrom utils import CID, schema\n\nfrom vec_index import hannoy \n\nimport os\nis_mini_instance = os.environ[\"M"
},
{
"path": "aquila/metricstore/router.py",
"chars": 3847,
"preview": "import logging\nlogging.basicConfig()\nlogging.getLogger().setLevel(logging.DEBUG)\n\nimport manager\n\nimport os\nimport json\n"
},
{
"path": "aquila/metricstore/run_tests.sh",
"chars": 181,
"preview": "rm -r /data/*\n\npython3 -m unittest test.apis.db_fns -v\npython3 -m unittest test.apis.doc_fns -v\npython3 -m unittest test"
},
{
"path": "aquila/metricstore/test/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "aquila/metricstore/test/apis/auth_fns.py",
"chars": 1965,
"preview": "import unittest\n\nimport index\nfrom utils import CID, schema\n\nfrom Crypto.Hash import SHA384\nfrom Crypto.PublicKey import"
},
{
"path": "aquila/metricstore/test/apis/db_fns.py",
"chars": 2063,
"preview": "import unittest\n\nimport index # includes preloading databases\nimport router\nfrom utils import CID, schema\n\nclass TestDB "
},
{
"path": "aquila/metricstore/test/apis/doc_fns.py",
"chars": 8587,
"preview": "import unittest\n\nimport index # includes preloading databases\nimport router\nfrom utils import CID\nimport numpy as np\n\nim"
},
{
"path": "aquila/metricstore/test/apis/search_fns.py",
"chars": 2174,
"preview": "import unittest\n\nimport time\n\nimport index # includes preloading databases\nimport router\nfrom utils import CID\n\nimport n"
},
{
"path": "aquila/metricstore/utils/CID.py",
"chars": 837,
"preview": "import logging\n\nimport bson\nimport hashlib\nimport base58\n\ndef doc2CID (inp):\n try:\n # JSON OBJ to BSON encode\n"
},
{
"path": "aquila/metricstore/utils/config.py",
"chars": 1789,
"preview": "import yaml\nimport os\n\nos.environ[\"DATA_STORE_LOCATION\"] = \"/data/\"\nos.environ[\"LOG_CHUNK_LEN\"] = \"1000\"\n\nif \"MINI_AQDB\""
},
{
"path": "aquila/metricstore/utils/cryptops.py",
"chars": 787,
"preview": "import logging\n\nfrom Crypto.Hash import SHA384\nfrom Crypto.PublicKey import RSA\nfrom Crypto.Signature import pkcs1_15\n\ni"
},
{
"path": "aquila/metricstore/utils/schema.py",
"chars": 2877,
"preview": "import logging\n\nimport fastjsonschema\n\ndef generate_schema (template):\n # get all keys from template schema\n metad"
},
{
"path": "aquila/metricstore/vec_index/hannoy.py",
"chars": 7526,
"preview": "import logging\n\nimport numpy as np\nfrom annoy import AnnoyIndex\nimport yaml\nimport os\n\nimport threading\nimport queue\nimp"
},
{
"path": "aquila/metricstore/vec_index/hfaiss.py",
"chars": 5971,
"preview": "import logging\n\nimport numpy as np\nimport faiss\nimport yaml\n\nimport os\nimport threading\nimport queue\nimport time\n\nclass "
},
{
"path": "aquila/metricstore/wal/walman.py",
"chars": 2455,
"preview": "import logging\n\nimport os\nimport time\nimport queue\nimport threading\n\nLOG_STORE_LOCATION = os.environ[\"DATA_STORE_LOCATIO"
},
{
"path": "aquila/metricstore/wal/walmon.py",
"chars": 0,
"preview": ""
},
{
"path": "aquila/network/go.mod",
"chars": 143,
"preview": "module aquilaport\n\ngo 1.15\n\nrequire (\n\tgithub.com/gorilla/mux v1.8.0\n\tgithub.com/syndtr/goleveldb v1.0.0\n\tgo.mongodb.org"
},
{
"path": "aquila/network/go.sum",
"chars": 11726,
"preview": "github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/aws/aws-sdk-go v1.34"
},
{
"path": "aquila/network/main.go",
"chars": 8603,
"preview": "// main.go\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"log\"\n\t\"net/http\"\n\t\"net/http/cookiejar\"\n\t\"string"
},
{
"path": "aquila/search/.dockerignore",
"chars": 31,
"preview": "node_modules\n.vscode\n.env*\ndist"
},
{
"path": "aquila/search/.nvmrc",
"chars": 8,
"preview": "v16.17.0"
},
{
"path": "aquila/search/Dockerfile",
"chars": 1389,
"preview": "FROM node:16-alpine as builder\n\n# Browserless\nARG BROWSERLESS_API_KEY=your-api-key-here\n\nWORKDIR /app\n\nRUN apk add curl "
},
{
"path": "aquila/search/README.md",
"chars": 1229,
"preview": "# AquilaX EE\n\n## ⚠️ Server and other Credentials are in workflow file\nNever make this repo Public\n\nSteps to run this pro"
},
{
"path": "aquila/search/package.json",
"chars": 1629,
"preview": "{\n \"name\": \"aquilax-ee\",\n \"version\": \"1.0.0\",\n \"description\": \"AquilaX api\",\n \"main\": \"index.js\",\n \"repository"
},
{
"path": "aquila/search/src/@types/express/index.d.ts",
"chars": 239,
"preview": "import { AccountStatus } from \"../../service/dto/AuthServiceDto\";\n\ndeclare global {\n\tnamespace Express {\n\t\tinterface Req"
},
{
"path": "aquila/search/src/@types/index.d.ts",
"chars": 880,
"preview": "declare module '@postlight/parser' {\n export type ExtractorArgs = any;\n\n export type ParserOptions = {\n html?: stri"
},
{
"path": "aquila/search/src/app.ts",
"chars": 1563,
"preview": "import 'reflect-metadata';\nimport express from 'express';\nimport { Container } from 'typedi';\nimport { useExpressServer,"
},
{
"path": "aquila/search/src/config/db.ts",
"chars": 703,
"preview": "import { DataSource, DataSourceOptions } from \"typeorm\";\n\nimport { ConfigService } from \"../lib/ConfigService\";\n\nconst c"
},
{
"path": "aquila/search/src/config/redisConnection.ts",
"chars": 388,
"preview": "import Redis from 'ioredis';\nimport { ConfigService } from '../lib/ConfigService';\n\nconst configService = new ConfigServ"
},
{
"path": "aquila/search/src/controller/AuthController.ts",
"chars": 598,
"preview": "import { Body, JsonController, Post } from \"routing-controllers\";\nimport { Service } from \"typedi\";\nimport { AuthService"
},
{
"path": "aquila/search/src/controller/BookmarkController.ts",
"chars": 3873,
"preview": "import { Body, Get, JsonController, Param, Post, QueryParams, UseBefore } from \"routing-controllers\";\nimport { Service }"
},
{
"path": "aquila/search/src/controller/CollectionController.ts",
"chars": 2838,
"preview": "import { Authorized, Get, JsonController, Param, QueryParams, UseBefore } from \"routing-controllers\";\nimport { Service }"
},
{
"path": "aquila/search/src/controller/CollectionSubscriptionController.ts",
"chars": 3987,
"preview": "import { Body, Get, JsonController, Param, Post, QueryParams, UseBefore } from \"routing-controllers\";\nimport { Service }"
},
{
"path": "aquila/search/src/controller/CustomerController.ts",
"chars": 2799,
"preview": "import { Authorized, Body, CurrentUser, Get, JsonController, Param, Patch, Post, Put, UseBefore } from \"routing-controll"
},
{
"path": "aquila/search/src/controller/HomeController.ts",
"chars": 523,
"preview": "import { Get, JsonController } from \"routing-controllers\";\nimport { Service } from \"typedi\";\nimport { Queue } from 'bull"
},
{
"path": "aquila/search/src/controller/dto/AuthControllerDto.ts",
"chars": 202,
"preview": "import { AccountStatus } from \"../../service/dto/AuthServiceDto\";\n\nexport interface LoginCustomerRequestBodyDto {\n\tsecre"
},
{
"path": "aquila/search/src/controller/dto/BookmarkControllerDto.ts",
"chars": 920,
"preview": "export interface AddBookmarkReqBodyDto {\n\thtml?: string;\n\turl: string;\n\tcollectionId: string;\n}\n\nexport interface GetBoo"
},
{
"path": "aquila/search/src/controller/dto/CollectionControllerDto.ts",
"chars": 681,
"preview": "import { Collection } from \"../../entity/Collection\";\nimport { CollectionTemp } from \"../../entity/CollectionTemp\";\n\nexp"
},
{
"path": "aquila/search/src/controller/dto/CollectionSubscriptionControllerDto.ts",
"chars": 429,
"preview": "export interface GetSubscriptionsByCustomerIdReqQueryParamsDto {\n\tquery?: string;\n\tlimit?: string;\n\tpage?: string;\n}\n\nex"
},
{
"path": "aquila/search/src/controller/dto/CustomerControllerDto.ts",
"chars": 837,
"preview": "import { CollectionTemp } from \"../../entity/CollectionTemp\";\nimport { CustomerTemp } from \"../../entity/CustomerTemp\";\n"
},
{
"path": "aquila/search/src/entity/Bookmark.ts",
"chars": 1377,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeorm\";\n\nexpor"
},
{
"path": "aquila/search/src/entity/BookmarkPara.ts",
"chars": 485,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeorm\";\n\n@Enti"
},
{
"path": "aquila/search/src/entity/BookmarkParaTemp.ts",
"chars": 494,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeorm\";\n\n@Enti"
},
{
"path": "aquila/search/src/entity/BookmarkTemp.ts",
"chars": 1415,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeorm\";\n\nexpor"
},
{
"path": "aquila/search/src/entity/Collection.ts",
"chars": 1193,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, JoinColumn, JoinTable, ManyToOne, PrimaryGeneratedColumn, UpdateD"
},
{
"path": "aquila/search/src/entity/CollectionSubscription.ts",
"chars": 675,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeorm\";\n\n@Enti"
},
{
"path": "aquila/search/src/entity/CollectionSubscriptionTemp.ts",
"chars": 684,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeorm\";\n\n@Enti"
},
{
"path": "aquila/search/src/entity/CollectionTemp.ts",
"chars": 909,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeorm\";\n\n@Enti"
},
{
"path": "aquila/search/src/entity/Customer.ts",
"chars": 1251,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, Generated, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeo"
},
{
"path": "aquila/search/src/entity/CustomerTemp.ts",
"chars": 1043,
"preview": "import { BaseEntity, Column, CreateDateColumn, Entity, Generated, PrimaryGeneratedColumn, UpdateDateColumn } from \"typeo"
},
{
"path": "aquila/search/src/helper/auth.ts",
"chars": 386,
"preview": "import { Action } from \"routing-controllers\";\nimport Container from \"typedi\";\nimport { AuthService } from \"../service/Au"
},
{
"path": "aquila/search/src/helper/decorators/jwtPayloadData.ts",
"chars": 477,
"preview": "import { Action, createParamDecorator } from \"routing-controllers\";\nimport Container from \"typedi\";\nimport { AuthService"
},
{
"path": "aquila/search/src/helper/user.ts",
"chars": 752,
"preview": "import { Action } from \"routing-controllers\";\nimport Container from \"typedi\";\nimport { Customer } from \"../entity/Custom"
},
{
"path": "aquila/search/src/index.ts",
"chars": 343,
"preview": "import { exit } from 'process';\nimport main from './app';\nimport { ConfigService } from './lib/ConfigService';\n\nconst co"
},
{
"path": "aquila/search/src/job/AppQueue.ts",
"chars": 395,
"preview": "import { Job, Queue } from \"bullmq\";\nimport { Service } from \"typedi\";\nimport connection from '../config/redisConnection"
},
{
"path": "aquila/search/src/job/appWorker.ts",
"chars": 257,
"preview": "import { Worker } from \"bullmq\";\nimport Redis from \"ioredis\";\n\nexport function getAppWorker(connection: Redis) {\n\tconst "
},
{
"path": "aquila/search/src/job/appWorkerProcessor.ts",
"chars": 4309,
"preview": "import { Job } from \"bullmq\";\nimport Parser from \"@postlight/parser\";\nimport axios from 'axios';\nimport Container from \""
},
{
"path": "aquila/search/src/job/types.ts",
"chars": 407,
"preview": "import { Bookmark } from \"../entity/Bookmark\"\nimport { BookmarkTemp } from \"../entity/BookmarkTemp\"\nimport { AccountStat"
},
{
"path": "aquila/search/src/lib/AquilaClientService.ts",
"chars": 1661,
"preview": "import { AquilaClient, Db, Hub, Schema, Wallet } from \"aquila-js\";\nimport path from \"path\";\nimport { Service } from \"typ"
},
{
"path": "aquila/search/src/lib/ConfigService.ts",
"chars": 968,
"preview": "import fs from 'fs';\nimport dotenv from 'dotenv';\nimport { Service } from 'typedi';\n\n@Service()\nexport class ConfigServi"
},
{
"path": "aquila/search/src/middleware/global/AuthMiddleware.ts",
"chars": 557,
"preview": "import { NextFunction, Request, Response } from 'express';\nimport { ExpressMiddlewareInterface } from 'routing-controlle"
},
{
"path": "aquila/search/src/middleware/global/GlobalErrorMiddleware.ts",
"chars": 1123,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { ExpressErrorMiddlewareInterface, HttpError, Middlewa"
},
{
"path": "aquila/search/src/middleware/global/TokenParserMiddleware.ts",
"chars": 712,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { ExpressMiddlewareInterface, Middleware } from \"routi"
},
{
"path": "aquila/search/src/middleware/validator/bookmark/AddBookmarkValidator.ts",
"chars": 2154,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { body } from \"express-validator\";\nimport { ExpressMid"
},
{
"path": "aquila/search/src/middleware/validator/bookmark/GetBookmarkByCollectionIdValidator.ts",
"chars": 1656,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { param, query } from \"express-validator\";\nimport { Ex"
},
{
"path": "aquila/search/src/middleware/validator/bookmark/GetFeaturedBookmarkValidator.ts",
"chars": 924,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { query } from \"express-validator\";\nimport { ExpressMi"
},
{
"path": "aquila/search/src/middleware/validator/bookmark/GetPublicBookmarkByCollectionIdParamValidator.ts",
"chars": 884,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { param, query } from \"express-validator\";\nimport { Ex"
},
{
"path": "aquila/search/src/middleware/validator/bookmark/GetPublicBookmarkByCollectionIdValidator.ts",
"chars": 1292,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { param, query } from \"express-validator\";\nimport { Ex"
},
{
"path": "aquila/search/src/middleware/validator/collection/GetAllFeaturedCollectionValidator.ts",
"chars": 823,
"preview": "import { NextFunction, Request } from \"express\";\nimport { query } from \"express-validator\";\nimport { ExpressMiddlewareIn"
},
{
"path": "aquila/search/src/middleware/validator/collection/GetAllPublicCollectionValidator.ts",
"chars": 821,
"preview": "import { NextFunction, Request } from \"express\";\nimport { query } from \"express-validator\";\nimport { ExpressMiddlewareIn"
},
{
"path": "aquila/search/src/middleware/validator/collection/GetPublicCollectionByIdValidator.ts",
"chars": 580,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { param } from \"express-validator\";\nimport { ExpressMi"
},
{
"path": "aquila/search/src/middleware/validator/csubscription/SubscribeCollectionValidator.ts",
"chars": 1341,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { BadRequestError, ExpressMiddlewareInterface } from \""
},
{
"path": "aquila/search/src/middleware/validator/csubscription/UnSubscribeCollectionValidator.ts",
"chars": 667,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { param } from \"express-validator\";\nimport { ExpressMi"
},
{
"path": "aquila/search/src/middleware/validator/customer/ActivateCustomerValidator.ts",
"chars": 2129,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { body } from \"express-validator\";\nimport { ExpressMid"
},
{
"path": "aquila/search/src/middleware/validator/customer/CreateCustomerValidator.ts",
"chars": 1233,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { body } from \"express-validator\";\nimport { ExpressMid"
},
{
"path": "aquila/search/src/middleware/validator/customer/GetCustomerPublicInfoByIdValidator.ts",
"chars": 578,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { param } from \"express-validator\";\nimport { ExpressMi"
},
{
"path": "aquila/search/src/middleware/validator/customer/UpdateCustomerValidator.ts",
"chars": 2570,
"preview": "import { NextFunction, Request, Response } from \"express\";\nimport { body } from \"express-validator\";\nimport { ExpressMid"
},
{
"path": "aquila/search/src/migrations/1676714378990-bookmark.ts",
"chars": 7108,
"preview": "import { MigrationInterface, QueryRunner } from \"typeorm\";\n\nexport class bookmark1676714378990 implements MigrationInter"
},
{
"path": "aquila/search/src/service/AuthService.ts",
"chars": 2391,
"preview": "import jwt, { Jwt } from 'jsonwebtoken';\nimport { BadRequestError, UnauthorizedError } from 'routing-controllers';\nimpor"
},
{
"path": "aquila/search/src/service/BookmarkService.ts",
"chars": 12910,
"preview": "import { BadRequestError, InternalServerError } from \"routing-controllers\";\nimport { Service } from \"typedi\";\nimport { I"
},
{
"path": "aquila/search/src/service/CollectionService.ts",
"chars": 4267,
"preview": "import { NotFoundError } from \"routing-controllers\";\nimport { Service } from \"typedi\";\n\nimport { Collection } from \"../e"
},
{
"path": "aquila/search/src/service/CollectionSubscriptionService.ts",
"chars": 14361,
"preview": "import { InternalServerError, NotFoundError } from \"routing-controllers\";\nimport { Service } from \"typedi\";\nimport { In "
},
{
"path": "aquila/search/src/service/CustomerService.ts",
"chars": 8680,
"preview": "import { Service } from \"typedi\";\nimport randomAnimalName from '../utils/randomAnimals';\nimport crypto from 'crypto';\nim"
},
{
"path": "aquila/search/src/service/dto/AuthServiceDto.ts",
"chars": 163,
"preview": "export enum AccountStatus {\n\tTEMPORARY= 'TEMPORARY',\n\tPERMANENT = 'PERMANENT'\n}\n\nexport interface JwtPayload {\n\tcustomer"
},
{
"path": "aquila/search/src/service/dto/BookmarkServiceDto.ts",
"chars": 1029,
"preview": "import { Bookmark } from \"../../entity/Bookmark\";\nimport { BookmarkTemp } from \"../../entity/BookmarkTemp\";\n\nexport inte"
},
{
"path": "aquila/search/src/service/dto/CollectionServiceDto.ts",
"chars": 441,
"preview": "import { Collection } from \"../../entity/Collection\";\nimport { CollectionTemp } from \"../../entity/CollectionTemp\";\n\nexp"
},
{
"path": "aquila/search/src/service/dto/CollectionSubscriptionServiceDto.ts",
"chars": 459,
"preview": "export interface GetSubscriptionsByCustomerIdOptionsInputDto {\n\tlimit: number;\n\tpage: number;\n\tquery?: string;\n}\n\nexport"
},
{
"path": "aquila/search/src/service/dto/CustomerServiceDto.ts",
"chars": 846,
"preview": "import { CollectionTemp } from \"../../entity/CollectionTemp\";\nimport { CustomerTemp } from \"../../entity/CustomerTemp\";\n"
},
{
"path": "aquila/search/src/utils/errors/ValidationError.ts",
"chars": 273,
"preview": "import { HttpError } from \"routing-controllers\";\n\nexport class ValidationError extends HttpError {\n\tconstructor(message:"
},
{
"path": "aquila/search/src/utils/randomAnimals.ts",
"chars": 3817,
"preview": "const adjectives = [\"furry\",\"ferocious\",\"dangerous\",\"tame\",\"agile\",\"clever\",\"aggressive\",\"tiny\",\"domestic\",\"wild\",\"herbi"
},
{
"path": "aquila/search/src/utils/validate.ts",
"chars": 514,
"preview": "import { Request } from \"express\";\nimport { ValidationChain, validationResult } from \"express-validator\";\nimport { Valid"
},
{
"path": "aquila/search/tsconfig.json",
"chars": 11043,
"preview": "{\n \"compilerOptions\": {\n /* Visit https://aka.ms/tsconfig.json to read more about this file */\n\n /* Projects */\n "
},
{
"path": "aquila/txt_transform/Dockerfile_mercury",
"chars": 176,
"preview": "FROM node:12.18.1\nENV NODE_ENV=production\n\nWORKDIR /app\n\nCOPY [\"package.json\", \"package-lock.json*\", \"./\"]\n\nRUN npm inst"
},
{
"path": "aquila/txt_transform/Dockerfile_txtpick",
"chars": 203,
"preview": "FROM python:3.8-slim-buster\n\nWORKDIR /app\n\nCOPY requirements.txt requirements.txt\nRUN pip3 install -r requirements.txt\n\n"
},
{
"path": "aquila/txt_transform/app.py",
"chars": 2991,
"preview": "# -*- coding: utf-8 -*-\n\nfrom __future__ import absolute_import\nfrom __future__ import division, print_function, unicode"
},
{
"path": "aquila/txt_transform/package.json",
"chars": 350,
"preview": "{\n \"name\": \"mercury\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"main\": \"index.js\",\n \"type\": \"module\",\n \"scripts\":"
},
{
"path": "aquila/txt_transform/requirements.txt",
"chars": 97,
"preview": "sumy==0.9.0\nrequests==2.27.1\nFlask_Cors==3.0.10\nFlask==2.0.3\nbeautifulsoup4==4.10.0\nnumpy==1.22.2"
},
{
"path": "aquila/txt_transform/server.js",
"chars": 693,
"preview": "import express from \"express\";\nvar app = express();\napp.use(express.json({limit: '50mb'}));\nconst port = 5009\n\nimport Me"
},
{
"path": "aquila/txt_transform/test.html",
"chars": 666335,
"preview": "<div><div id=\"mw-content-text\" class=\"mw-body-content mw-content-ltr\"><div class=\"mw-parser-output\"><div class=\"shortdes"
},
{
"path": "aquila/txt_transform/test.py",
"chars": 1415,
"preview": "import requests\nimport json\n\ndef process_html (html, url):\n payload = {\n \"html\": html,\n \"url\": url\n "
},
{
"path": "aquila/view/.dockerignore",
"chars": 24,
"preview": "node_modules\n.env*\n.next"
},
{
"path": "aquila/view/.eslintrc.json",
"chars": 40,
"preview": "{\n \"extends\": \"next/core-web-vitals\"\n}\n"
},
{
"path": "aquila/view/@types/next-auth.d.ts",
"chars": 748,
"preview": "import { DefaultSession } from \"next-auth\";\n\ndeclare module \"next-auth\" {\n interface User {\n customerId: strin"
},
{
"path": "aquila/view/Dockerfile",
"chars": 661,
"preview": "FROM node:16-alpine as builder\n\nARG NEXTAUTH_URL\nARG NEXTAUTH_SECRET\nARG NEXT_PUBLIC_AQUILA_API_URL\nARG NEXT_PUBLIC_BASE"
},
{
"path": "aquila/view/README.md",
"chars": 483,
"preview": "# Aquila-Network-UI\n\n\n# ⚠️ Server and other Credentials are in workflow file.\nNever Make this Repo Public\n\n## Getting St"
},
{
"path": "aquila/view/components/hoc/InitComponent.tsx",
"chars": 1127,
"preview": "import { signOut, useSession, getSession } from \"next-auth/react\";\nimport { FC, useEffect } from \"react\";\nimport { useAp"
},
{
"path": "aquila/view/components/layout/base/baseLayout.tsx",
"chars": 406,
"preview": "import { FC } from 'react';\n\nimport { ProgressLoaderProvider } from '../../ui/progressLoader/ProgressLoader';\n\n\ninterfac"
},
{
"path": "aquila/view/components/layout/boxCenter/BoxCenterLayout.module.scss",
"chars": 371,
"preview": ".box-center-layout {\n\tdisplay: flex;\n\talign-items: center;\n\tjustify-content: center;\n\theight: 100vh;\n\n\t&__box {\n\t\tbackgr"
},
{
"path": "aquila/view/components/layout/boxCenter/BoxCenterLayout.tsx",
"chars": 802,
"preview": "import Img from 'next/image';\nimport Link from 'next/link';\n\nimport Logo from '../../../public/img/logo.png';\nimport Bas"
},
{
"path": "aquila/view/components/layout/childLayout/settings/Header.module.scss",
"chars": 313,
"preview": ".header {\n display: flex;\n gap: 15px;\n align-items: center;\n padding: 10px 0px;\n\n &__info {\n color"
},
{
"path": "aquila/view/components/layout/childLayout/settings/Header.tsx",
"chars": 823,
"preview": "import Avatar from 'boring-avatars';\nimport { FC } from 'react';\nimport { AppState } from '../../../../store';\n\nimport c"
},
{
"path": "aquila/view/components/layout/childLayout/settings/SettingsLayout.module.scss",
"chars": 255,
"preview": ".settings {\n width: 900px;\n margin: auto;\n margin-top: 20px;\n\n &__header {\n }\n\n &__body {\n disp"
},
{
"path": "aquila/view/components/layout/childLayout/settings/SettingsLayout.tsx",
"chars": 1071,
"preview": "import { FC } from \"react\";\nimport { AppState } from \"../../../../store\";\nimport MainLayout from \"../../main/MainLayout\""
},
{
"path": "aquila/view/components/layout/childLayout/settings/Sidebar.module.scss",
"chars": 309,
"preview": ".sidebar {\n list-style: none;\n margin: 0px;\n padding: 0px;\n\n &__item {\n padding: 10px 5px;\n }\n\n "
},
{
"path": "aquila/view/components/layout/childLayout/settings/Sidebar.tsx",
"chars": 639,
"preview": "import Link from \"next/link\";\n\nimport classes from \"./Sidebar.module.scss\"; \n\nconst Sidebar = () => {\n return (\n "
},
{
"path": "aquila/view/components/layout/main/AddLink.module.scss",
"chars": 1521,
"preview": ".add-link {\n max-width: 500px;\n box-sizing: border-box;\n padding: 10px;\n box-shadow: rgba(0, 0, 0, 0.16) 0px"
},
{
"path": "aquila/view/components/layout/main/AddLink.tsx",
"chars": 2386,
"preview": "import { FC, useEffect } from 'react';\nimport { IoCloseCircleOutline } from 'react-icons/io5';\nimport { useForm } from '"
},
{
"path": "aquila/view/components/layout/main/Footer.module.scss",
"chars": 868,
"preview": ".footer {\n padding: 20px 0px;\n text-align: center;\n box-sizing: border-box;\n\n &__icon-menu {\n list-st"
},
{
"path": "aquila/view/components/layout/main/Footer.tsx",
"chars": 1955,
"preview": "import { FC, useState } from 'react';\nimport Link from 'next/link';\nimport { FaGithub, FaYoutube, FaMedium } from 'react"
},
{
"path": "aquila/view/components/layout/main/Header.module.scss",
"chars": 3491,
"preview": ".header {\n padding: 20px 0px;\n\n &__container {\n margin: auto;\n display: flex;\n justify-conten"
},
{
"path": "aquila/view/components/layout/main/Header.tsx",
"chars": 5800,
"preview": "import Image from \"next/image\";\nimport Link from \"next/link\";\nimport { FC, useState } from \"react\";\nimport Avatar from '"
},
{
"path": "aquila/view/components/layout/main/MainLayout.module.scss",
"chars": 178,
"preview": ".main-layout {\n\theight: 100%;\n\tdisplay: flex;\n\tflex-direction: column;\n\tjustify-content: space-between;\n\n\t&__header--wit"
},
{
"path": "aquila/view/components/layout/main/MainLayout.tsx",
"chars": 3564,
"preview": "import { signOut } from 'next-auth/react';\nimport { useRouter } from 'next/router';\nimport { FC, useEffect, useState } f"
},
{
"path": "aquila/view/components/pages/account/editProfile/EditProfileForm.module.scss",
"chars": 897,
"preview": ".edit-profile-form {\n &__form-group {\n margin-bottom: 15px;\n }\n\n &__form-label {\n display: block;"
},
{
"path": "aquila/view/components/pages/account/editProfile/EditProfileForm.tsx",
"chars": 6356,
"preview": "import { FC, useEffect } from 'react';\nimport { useForm } from 'react-hook-form';\nimport { AppState } from '../../../../"
},
{
"path": "aquila/view/components/pages/account/editProfile/EditProfileHeader.module.scss",
"chars": 81,
"preview": ".header {\n &__title {\n font-size: 22px;\n color: #202A38;\n }\n}"
},
{
"path": "aquila/view/components/pages/account/editProfile/EditProfileHeader.tsx",
"chars": 275,
"preview": "import classes from './EditProfileHeader.module.scss';\n\nconst EditProfileHeader = () => {\n return (\n <header c"
},
{
"path": "aquila/view/components/pages/account/editProfile/EditProfileWrapper.module.scss",
"chars": 108,
"preview": ".edit-profile-wrapper {\n margin-top: 20px;\n\n &__alert-container {\n margin-bottom: 20px;\n }\n}"
},
{
"path": "aquila/view/components/pages/account/editProfile/EditProfileWrapper.tsx",
"chars": 2168,
"preview": "import { FC } from 'react';\nimport { Oval } from 'react-loader-spinner';\nimport { AppState } from '../../../../store';\n\n"
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/CollectionBookmarksPageWrapper.module.scss",
"chars": 1063,
"preview": ".collection-bookmarks {\n display: flex;\n gap: 50px;\n\n &__search-area {\n flex-basis: 900px; \n }\n\n "
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/CollectionBookmarksPageWrapper.tsx",
"chars": 4300,
"preview": "import { FC, useEffect, useState } from \"react\";\nimport { Oval } from 'react-loader-spinner';\nimport { FiX } from 'react"
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchBar.module.scss",
"chars": 336,
"preview": ".search-bar {\n\tpadding: 10px 5px;\n\n\t&__container {\n\t\tdisplay: flex;\n\t\tborder-bottom: 1px solid #AEBCD1;\n\t\tpadding: 10px "
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchBar.tsx",
"chars": 813,
"preview": "import { FC, useRef } from 'react';\nimport { IoSearch } from 'react-icons/io5';\nimport classes from './SearchBar.module."
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchPageProfile.module.scss",
"chars": 2041,
"preview": ".search-profile {\n\tpadding: 15px;\n\tmargin-top: 20px;\n\tborder: 1px solid #CED7E3;\n\tbackground: #fff;\n\tborder-radius: 3px;"
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchPageProfile.tsx",
"chars": 3533,
"preview": "import { VscFeedback } from 'react-icons/vsc';\nimport { toast } from 'react-toastify';\nimport Avatar from \"boring-avatar"
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResultItem.module.scss",
"chars": 647,
"preview": ".search-result-item {\n\tborder-bottom: 1px solid #CED7E3;\n\tpadding-bottom: 15px;\n\tmargin-bottom: 10px;\n\tword-wrap: break-"
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResultItem.tsx",
"chars": 924,
"preview": "import moment from 'moment';\nimport { FC } from 'react';\nimport classes from './SearchResultItem.module.scss';\n\ninterfac"
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResults.module.scss",
"chars": 437,
"preview": ".search-result {\n\n\t&__results {\n\t\tmargin-top: 20px;\n\t}\n\n\t&__item {\n\t\tmargin-bottom: 25px;\n\t}\n\n\t&__header-info-desc {\n\t\tc"
},
{
"path": "aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResults.tsx",
"chars": 1704,
"preview": "import { FC } from \"react\";\nimport { Bookmark } from \"../../../../../store/slices/types/Bookmark\";\nimport SearchResultIt"
},
{
"path": "aquila/view/components/pages/explore/Explore.module.scss",
"chars": 792,
"preview": ".explore {\n\t// width: 250px;\n\tpadding: 20px 15px;\n\tborder-radius: 3px;\n\tborder: 1px solid #ddd;\n\n\t&__header {\n\t\tdisplay:"
},
{
"path": "aquila/view/components/pages/explore/Explore.tsx",
"chars": 975,
"preview": "import Avatar from \"boring-avatars\";\nimport Link from \"next/link\";\nimport { FC } from \"react\";\nimport { Collection } fro"
},
{
"path": "aquila/view/components/pages/explore/ExploreCategoryItem.module.scss",
"chars": 644,
"preview": ".explore-category-item {\n\tmargin-top: 15px;\n\n\t&__title {\n\t\tmargin-bottom: 20px;\n\t\tfont-size: 24px;\n\t}\n\n\t&__explore-list "
},
{
"path": "aquila/view/components/pages/explore/ExploreCategoryItem.tsx",
"chars": 794,
"preview": "import { FC } from \"react\";\nimport { Collection } from \"../../../store/slices/types/Collection\";\nimport Explore from \"./"
},
{
"path": "aquila/view/components/pages/explore/ExploreCategoryList.module.scss",
"chars": 63,
"preview": ".explore-category-list {\n\t&__item {\n\t\tmargin-bottom: 50px;\n\t}\n}"
},
{
"path": "aquila/view/components/pages/explore/ExploreCategoryList.tsx",
"chars": 1091,
"preview": "import { FC } from \"react\";\nimport { AppState } from \"../../../store\";\nimport ExploreCategoryItem from \"./ExploreCategor"
},
{
"path": "aquila/view/components/pages/explore/ExplorePageWrapper.tsx",
"chars": 774,
"preview": "import { FC } from \"react\";\nimport { AppState } from \"../../../store\";\nimport MainLayout from \"../../layout/main/MainLay"
},
{
"path": "aquila/view/components/pages/home/HomePageWrapper.module.scss",
"chars": 1034,
"preview": ".home-page {\n display: flex;\n gap: 50px;\n\n &__search-area {\n flex-basis: 800px; \n }\n\n &__search"
},
{
"path": "aquila/view/components/pages/home/HomePageWrapper.tsx",
"chars": 3712,
"preview": "import { FC, useEffect, useState } from \"react\";\nimport { Oval } from 'react-loader-spinner';\nimport { FiX } from 'react"
},
{
"path": "aquila/view/components/pages/home/SearchBar.module.scss",
"chars": 336,
"preview": ".search-bar {\n\tpadding: 10px 5px;\n\n\t&__container {\n\t\tdisplay: flex;\n\t\tborder-bottom: 1px solid #AEBCD1;\n\t\tpadding: 10px "
},
{
"path": "aquila/view/components/pages/home/SearchBar.tsx",
"chars": 813,
"preview": "import { FC, useRef } from 'react';\nimport { IoSearch } from 'react-icons/io5';\nimport classes from './SearchBar.module."
},
{
"path": "aquila/view/components/pages/home/SearchPageProfile.module.scss",
"chars": 1958,
"preview": ".search-profile {\n\tpadding: 15px;\n\tmargin-top: 20px;\n\tborder: 1px solid #CED7E3;\n\tborder-radius: 3px;\n\tbackground: #fff;"
},
{
"path": "aquila/view/components/pages/home/SearchPageProfile.tsx",
"chars": 3341,
"preview": "import { VscFeedback } from 'react-icons/vsc';\nimport { toast } from 'react-toastify';\nimport Avatar from \"boring-avatar"
},
{
"path": "aquila/view/components/pages/home/SearchResultItem.module.scss",
"chars": 663,
"preview": ".search-result-item {\n\tborder-bottom: 1px solid #CED7E3;\n\tpadding-bottom: 15px;\n\tmargin-bottom: 10px;\n\tword-wrap: break-"
},
{
"path": "aquila/view/components/pages/home/SearchResultItem.tsx",
"chars": 924,
"preview": "import { FC } from 'react';\nimport moment from 'moment';\n\nimport classes from './SearchResultItem.module.scss';\n\ninterfa"
},
{
"path": "aquila/view/components/pages/home/SearchResults.module.scss",
"chars": 437,
"preview": ".search-result {\n\n\t&__results {\n\t\tmargin-top: 20px;\n\t}\n\n\t&__item {\n\t\tmargin-bottom: 25px;\n\t}\n\n\t&__header-info-desc {\n\t\tc"
},
{
"path": "aquila/view/components/pages/home/SearchResults.tsx",
"chars": 1697,
"preview": "import { FC } from \"react\";\nimport { Bookmark } from \"../../../store/slices/types/Bookmark\";\nimport SearchResultItem fro"
},
{
"path": "aquila/view/components/pages/index/AllControl.module.scss",
"chars": 1056,
"preview": ".all-control {\n display: flex;\n margin: auto;\n max-width: 1280;\n\n &__img-area {\n flex-basis: 50%;\n "
},
{
"path": "aquila/view/components/pages/index/AllControl.tsx",
"chars": 882,
"preview": "import Image from \"next/image\";\n\nimport classes from \"./AllControl.module.scss\";\nimport AllControlImg from \"../../../pub"
},
{
"path": "aquila/view/components/pages/index/Discover.module.scss",
"chars": 934,
"preview": ".discover {\n margin: auto;\n margin-top: 80px;\n display: flex;\n max-width: 1280px;\n &__text-area {\n "
},
{
"path": "aquila/view/components/pages/index/Discover.tsx",
"chars": 781,
"preview": "import Image from 'next/image';\nimport classes from './Discover.module.scss';\n\nimport DiscoverImg from '../../../public/"
},
{
"path": "aquila/view/components/pages/index/Hero.module.scss",
"chars": 1641,
"preview": ".hero {\n text-align: center;\n\n &__text-area {\n margin-top: 50px;\n }\n\n &__text-main-title {\n fo"
},
{
"path": "aquila/view/components/pages/index/Hero.tsx",
"chars": 1289,
"preview": "import Image from 'next/image';\nimport classes from './Hero.module.scss';\n\nimport HeroImg from '../../../public/img/aqui"
},
{
"path": "aquila/view/components/pages/index/IndexPageWrapper.tsx",
"chars": 378,
"preview": "import MainLayout from \"../../layout/main/MainLayout\";\nimport AllControl from \"./AllControl\";\nimport Discover from \"./Di"
},
{
"path": "aquila/view/components/pages/index/Story.module.scss",
"chars": 384,
"preview": ".story {\n background: #EFF2F6;\n padding: 100px 0px;\n text-align: center;\n\n &__title {\n font-size: 24p"
},
{
"path": "aquila/view/components/pages/index/Story.tsx",
"chars": 319,
"preview": "import classes from './Story.module.scss';\n\nconst Story = () => {\n return (\n <div className={classes.story}>\n "
},
{
"path": "aquila/view/components/pages/signIn/SignInForm.module.scss",
"chars": 946,
"preview": ".login-box {\n\n\t&__header {\n\t\tmargin: 25px 0px;\n\t}\n\n\t&__title {\n\t\tfont-size: 20px;\n\t\tcolor: #415572;\n\t\ttext-align: center"
},
{
"path": "aquila/view/components/pages/signIn/SignInForm.tsx",
"chars": 1545,
"preview": "import Link from 'next/link';\nimport { useRef, useState } from 'react';\nimport classes from './SignInForm.module.scss';\n"
},
{
"path": "aquila/view/components/pages/signIn/SignInPageWrapper.tsx",
"chars": 398,
"preview": "import { FC } from \"react\";\nimport BoxCenterLayout from \"../../layout/boxCenter/BoxCenterLayout\";\nimport SignInForm from"
},
{
"path": "aquila/view/components/pages/signUp/SecretKey.module.scss",
"chars": 1531,
"preview": ".secret-key {\n padding: 20px 0px;\n text-align: center;\n\n &__header {\n padding: 5px 0px;\n }\n\n &__he"
}
]
// ... and 73 more files (download for full content)
About this extraction
This page contains the full source code of the Aquila-Network/AquilaDB GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 273 files (1.1 MB), approximately 334.5k tokens, and a symbol index with 510 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.