Repository: Aquila-Network/AquilaDB Branch: main Commit: b086fa65f5cc Files: 273 Total size: 1.1 MB Directory structure: gitextract_6lg6_8cc/ ├── README.md └── aquila/ ├── encoder/ │ ├── CODE_OF_CONDUCT.md │ ├── Dockerfile │ ├── LICENSE │ ├── README.md │ ├── authentication.py │ ├── config.yml │ ├── encoder.py │ ├── index.py │ ├── ipfs.service │ ├── manager.py │ ├── requirements.txt │ ├── run_tests.sh │ ├── test/ │ │ ├── __init__.py │ │ └── apis/ │ │ └── hub_fns.py │ ├── test.py │ └── utils/ │ ├── CID.py │ ├── config.py │ ├── cryptops.py │ ├── downloader.py │ └── schema.py ├── metricstore/ │ ├── .travis.yml │ ├── CODE_OF_CONDUCT.md │ ├── DB_config.yml │ ├── Dockerfile │ ├── DockerfileBig │ ├── LICENSE │ ├── README.md │ ├── authentication.py │ ├── index.py │ ├── install.sh │ ├── manager.py │ ├── router.py │ ├── run_tests.sh │ ├── test/ │ │ ├── __init__.py │ │ └── apis/ │ │ ├── auth_fns.py │ │ ├── db_fns.py │ │ ├── doc_fns.py │ │ └── search_fns.py │ ├── utils/ │ │ ├── CID.py │ │ ├── config.py │ │ ├── cryptops.py │ │ └── schema.py │ ├── vec_index/ │ │ ├── hannoy.py │ │ └── hfaiss.py │ └── wal/ │ ├── walman.py │ └── walmon.py ├── network/ │ ├── go.mod │ ├── go.sum │ └── main.go ├── search/ │ ├── .dockerignore │ ├── .nvmrc │ ├── Dockerfile │ ├── README.md │ ├── package.json │ ├── src/ │ │ ├── @types/ │ │ │ ├── express/ │ │ │ │ └── index.d.ts │ │ │ └── index.d.ts │ │ ├── app.ts │ │ ├── config/ │ │ │ ├── db.ts │ │ │ └── redisConnection.ts │ │ ├── controller/ │ │ │ ├── AuthController.ts │ │ │ ├── BookmarkController.ts │ │ │ ├── CollectionController.ts │ │ │ ├── CollectionSubscriptionController.ts │ │ │ ├── CustomerController.ts │ │ │ ├── HomeController.ts │ │ │ └── dto/ │ │ │ ├── AuthControllerDto.ts │ │ │ ├── BookmarkControllerDto.ts │ │ │ ├── CollectionControllerDto.ts │ │ │ ├── CollectionSubscriptionControllerDto.ts │ │ │ └── CustomerControllerDto.ts │ │ ├── entity/ │ │ │ ├── Bookmark.ts │ │ │ ├── BookmarkPara.ts │ │ │ ├── BookmarkParaTemp.ts │ │ │ ├── BookmarkTemp.ts │ │ │ ├── Collection.ts │ │ │ ├── CollectionSubscription.ts │ │ │ ├── CollectionSubscriptionTemp.ts │ │ │ ├── CollectionTemp.ts │ │ │ ├── Customer.ts │ │ │ └── CustomerTemp.ts │ │ ├── helper/ │ │ │ ├── auth.ts │ │ │ ├── decorators/ │ │ │ │ └── jwtPayloadData.ts │ │ │ └── user.ts │ │ ├── index.ts │ │ ├── job/ │ │ │ ├── AppQueue.ts │ │ │ ├── appWorker.ts │ │ │ ├── appWorkerProcessor.ts │ │ │ └── types.ts │ │ ├── lib/ │ │ │ ├── AquilaClientService.ts │ │ │ └── ConfigService.ts │ │ ├── middleware/ │ │ │ ├── global/ │ │ │ │ ├── AuthMiddleware.ts │ │ │ │ ├── GlobalErrorMiddleware.ts │ │ │ │ └── TokenParserMiddleware.ts │ │ │ └── validator/ │ │ │ ├── bookmark/ │ │ │ │ ├── AddBookmarkValidator.ts │ │ │ │ ├── GetBookmarkByCollectionIdValidator.ts │ │ │ │ ├── GetFeaturedBookmarkValidator.ts │ │ │ │ ├── GetPublicBookmarkByCollectionIdParamValidator.ts │ │ │ │ └── GetPublicBookmarkByCollectionIdValidator.ts │ │ │ ├── collection/ │ │ │ │ ├── GetAllFeaturedCollectionValidator.ts │ │ │ │ ├── GetAllPublicCollectionValidator.ts │ │ │ │ └── GetPublicCollectionByIdValidator.ts │ │ │ ├── csubscription/ │ │ │ │ ├── SubscribeCollectionValidator.ts │ │ │ │ └── UnSubscribeCollectionValidator.ts │ │ │ └── customer/ │ │ │ ├── ActivateCustomerValidator.ts │ │ │ ├── CreateCustomerValidator.ts │ │ │ ├── GetCustomerPublicInfoByIdValidator.ts │ │ │ └── UpdateCustomerValidator.ts │ │ ├── migrations/ │ │ │ └── 1676714378990-bookmark.ts │ │ ├── service/ │ │ │ ├── AuthService.ts │ │ │ ├── BookmarkService.ts │ │ │ ├── CollectionService.ts │ │ │ ├── CollectionSubscriptionService.ts │ │ │ ├── CustomerService.ts │ │ │ └── dto/ │ │ │ ├── AuthServiceDto.ts │ │ │ ├── BookmarkServiceDto.ts │ │ │ ├── CollectionServiceDto.ts │ │ │ ├── CollectionSubscriptionServiceDto.ts │ │ │ └── CustomerServiceDto.ts │ │ └── utils/ │ │ ├── errors/ │ │ │ └── ValidationError.ts │ │ ├── randomAnimals.ts │ │ └── validate.ts │ └── tsconfig.json ├── txt_transform/ │ ├── Dockerfile_mercury │ ├── Dockerfile_txtpick │ ├── app.py │ ├── package.json │ ├── requirements.txt │ ├── server.js │ ├── test.html │ └── test.py └── view/ ├── .dockerignore ├── .eslintrc.json ├── @types/ │ └── next-auth.d.ts ├── Dockerfile ├── README.md ├── components/ │ ├── hoc/ │ │ └── InitComponent.tsx │ ├── layout/ │ │ ├── base/ │ │ │ └── baseLayout.tsx │ │ ├── boxCenter/ │ │ │ ├── BoxCenterLayout.module.scss │ │ │ └── BoxCenterLayout.tsx │ │ ├── childLayout/ │ │ │ └── settings/ │ │ │ ├── Header.module.scss │ │ │ ├── Header.tsx │ │ │ ├── SettingsLayout.module.scss │ │ │ ├── SettingsLayout.tsx │ │ │ ├── Sidebar.module.scss │ │ │ └── Sidebar.tsx │ │ └── main/ │ │ ├── AddLink.module.scss │ │ ├── AddLink.tsx │ │ ├── Footer.module.scss │ │ ├── Footer.tsx │ │ ├── Header.module.scss │ │ ├── Header.tsx │ │ ├── MainLayout.module.scss │ │ └── MainLayout.tsx │ ├── pages/ │ │ ├── account/ │ │ │ └── editProfile/ │ │ │ ├── EditProfileForm.module.scss │ │ │ ├── EditProfileForm.tsx │ │ │ ├── EditProfileHeader.module.scss │ │ │ ├── EditProfileHeader.tsx │ │ │ ├── EditProfileWrapper.module.scss │ │ │ └── EditProfileWrapper.tsx │ │ ├── collection/ │ │ │ └── bookmark/ │ │ │ └── CollectionBookmarks/ │ │ │ ├── CollectionBookmarksPageWrapper.module.scss │ │ │ ├── CollectionBookmarksPageWrapper.tsx │ │ │ ├── SearchBar.module.scss │ │ │ ├── SearchBar.tsx │ │ │ ├── SearchPageProfile.module.scss │ │ │ ├── SearchPageProfile.tsx │ │ │ ├── SearchResultItem.module.scss │ │ │ ├── SearchResultItem.tsx │ │ │ ├── SearchResults.module.scss │ │ │ └── SearchResults.tsx │ │ ├── explore/ │ │ │ ├── Explore.module.scss │ │ │ ├── Explore.tsx │ │ │ ├── ExploreCategoryItem.module.scss │ │ │ ├── ExploreCategoryItem.tsx │ │ │ ├── ExploreCategoryList.module.scss │ │ │ ├── ExploreCategoryList.tsx │ │ │ └── ExplorePageWrapper.tsx │ │ ├── home/ │ │ │ ├── HomePageWrapper.module.scss │ │ │ ├── HomePageWrapper.tsx │ │ │ ├── SearchBar.module.scss │ │ │ ├── SearchBar.tsx │ │ │ ├── SearchPageProfile.module.scss │ │ │ ├── SearchPageProfile.tsx │ │ │ ├── SearchResultItem.module.scss │ │ │ ├── SearchResultItem.tsx │ │ │ ├── SearchResults.module.scss │ │ │ └── SearchResults.tsx │ │ ├── index/ │ │ │ ├── AllControl.module.scss │ │ │ ├── AllControl.tsx │ │ │ ├── Discover.module.scss │ │ │ ├── Discover.tsx │ │ │ ├── Hero.module.scss │ │ │ ├── Hero.tsx │ │ │ ├── IndexPageWrapper.tsx │ │ │ ├── Story.module.scss │ │ │ └── Story.tsx │ │ ├── signIn/ │ │ │ ├── SignInForm.module.scss │ │ │ ├── SignInForm.tsx │ │ │ └── SignInPageWrapper.tsx │ │ ├── signUp/ │ │ │ ├── SecretKey.module.scss │ │ │ ├── SecretKey.tsx │ │ │ ├── SignUpForm.module.scss │ │ │ ├── SignUpForm.tsx │ │ │ └── SignUpPageWrapper.tsx │ │ └── subscription/ │ │ ├── Collection.module.scss │ │ ├── Collection.tsx │ │ ├── HomePageWrapper.tsx │ │ ├── SearchBar.module.scss │ │ ├── SearchBar.tsx │ │ ├── SearchPageProfile.module.scss │ │ ├── SearchPageProfile.tsx │ │ ├── SearchResultItem.module.scss │ │ ├── SearchResultItem.tsx │ │ ├── SearchResults.module.scss │ │ ├── SearchResults.tsx │ │ ├── SubscribedCollections.module.scss │ │ ├── SubscribedCollections.tsx │ │ ├── SubscriptionPageWrapper.module.scss │ │ └── SubscriptionPageWrapper.tsx │ └── ui/ │ ├── alert/ │ │ ├── Alert.module.scss │ │ └── Alert.tsx │ ├── layout/ │ │ ├── Container.module.scss │ │ └── Container.tsx │ ├── modal/ │ │ ├── Modal.module.scss │ │ └── Modal.tsx │ └── progressLoader/ │ ├── ProgressLoader.module.scss │ └── ProgressLoader.tsx ├── middleware.ts ├── next.config.js ├── package.json ├── pages/ │ ├── _app.tsx │ ├── _document.tsx │ ├── account/ │ │ └── edit-profile.tsx │ ├── api/ │ │ ├── auth/ │ │ │ └── [...nextauth].ts │ │ └── hello.ts │ ├── collection/ │ │ └── bookmark/ │ │ └── [collectionId].tsx │ ├── explore.tsx │ ├── home.tsx │ ├── index.tsx │ ├── sign-in.tsx │ ├── sign-up.tsx │ ├── subscription.tsx │ └── test.tsx ├── store/ │ ├── index.ts │ └── slices/ │ ├── auth.ts │ ├── bookmark/ │ │ ├── addLink.ts │ │ ├── getLoggedInCustBookmarksByCollectionId.ts │ │ └── getPublicBookmarksByCollectionId.ts │ ├── collection/ │ │ ├── getAllPublicCollections.ts │ │ ├── getCollectionById.ts │ │ ├── getCustomerSubscriptions.ts │ │ ├── getFeaturedCollections.ts │ │ ├── getLoggedInCustCollections.ts │ │ ├── getSubscribedCollections.ts │ │ ├── isCollectionSubscribed.ts │ │ ├── subscribeCollectionById.ts │ │ └── unSubscribeCollectionById.ts │ ├── customer/ │ │ ├── activateCustomer.ts │ │ ├── getCurrentLoggedInCustomer.ts │ │ ├── getCustomerById.ts │ │ └── updateCustomer.ts │ ├── errors/ │ │ └── AsyncThunkSubmissionError.ts │ ├── generateName.ts │ ├── signup.ts │ ├── types/ │ │ ├── Bookmark.ts │ │ ├── Collection.ts │ │ ├── CollectionSubscription.ts │ │ ├── Customer.ts │ │ └── validationErrors.ts │ └── utils/ │ └── createError.ts ├── styles/ │ └── globals.scss ├── tsconfig.json └── utils/ └── api.ts ================================================ FILE CONTENTS ================================================ ================================================ FILE: README.md ================================================
Aquila Network Logo

Aquila DB

Easy to use Neural Search Engine


**[Aquila DB](https://github.com/Aquila-Network/AquilaDB)** is a Neural search engine. In other words, it is a database to index **Latent Vectors** generated by ML models along with **JSON Metadata** to perform **k-NN** retrieval. It is dead simple to set up, language-agnostic, and drop in addition to your Machine Learning Applications. Aquila DB, as of current features is a ready solution for Machine Learning engineers and Data scientists to build **[Neural Information Retrieval](https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf)** applications out of the box with minimal dependencies. > This project is still in alpha version & we're already using it in production to power semantic search at https://aquila.network. Wanna support this project? Yes, we love getting a **star** ⭐ and **shout-out** 🗣️ 🤗 Join [Community chat and get support: ![discord chatroom for discussions](https://www.freeiconspng.com/minicovers/flat-discord-material-like-icon--2.png)](https://discord.gg/5YP7zHS) # Who is this for * If you are working on a data science project and need to store a hell of a lot of data and retrieve similar data based on some feature vector, this will be a useful tool to you, with extra benefits a real world web application needs. * Are you dealing with a lot of images and related metadata? Want to find similar ones? You are at the right place. * If you are looking for a document database, this is not the right place for you. # Technology Aquila DB powers search features of Aquila Network. Here is where Aquila DB fits in the entire ecosystem:
Aquila DB Architecture
If you are serious and wanna dive down the rabbit hole, read our **[whitepapers](https://github.com/Aquila-Network/whitepaper)** and **[technical specifications](https://github.com/Aquila-Network/specs)** (being actively worked on). **As a side note**, everything in **[Aquila Network](https://github.com/Aquila-Network)** is defined by the specifications and a large chunk of our efforts goes into it. We also maintain quality implementations of those specifications with non-technical users in mind. This is to make sure that - Aquila Network is fully open, decentralized by design, and Fair. You can follow those specifications to implement your alternative software and still interact with the network without any restrictions. # Install ### Debian Run `curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/install.sh | /bin/bash -s -- -d 1 `. ### Docker **You need docker installed in your system** Build image (lite): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/Dockerfile -t aquiladb:local` Build image (big data): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/DockerfileBig -t aquiladb:localbg` Run image (to deploy Aquila DB lite): `docker run -p 5001:5001 -d aquiladb:local` Run image (to deploy Aquila DB big): `docker run -p 5001:5001 -d aquiladb:localbg` # Client SDKs We currently have multiple client libraries in progress to abstract the communication between deployed Aquila DB and your applications. [Python](https://github.com/Aquila-Network/AquilaPy) [Node JS](https://github.com/Aquila-Network/AquilaJS) ## Where to get private key (wallet key) for client authentication When you use a client library to authenticate with AquilaDB, you might need access the same private key (wallet key) used by AquilaDB. This key is located inside `/ossl/` directory within AquilaDB docker container (in your computer if you have installed AquilaDB directly without docker). To access the keys inside your AquilaDB container, follow below steps: * identify `CONTAINER ID` for the already running `aquiladb` docker instance: `docker ps` * take a copy of private keys from docker container to your host machine: `docker cp CONTAINER_ID:/ossl/ ./` * now you will see a new directory named `ossl` at your current location. Use the keys inside it. #### tips for advanced users If your pipeline requires the private keys to be generated in advance, you can do it in your host machine and then mount it to the container's `/ossl/` directory. Run: ``` mkdir -p /ossl/ openssl genrsa -passout pass:1234 -des3 -out /ossl/private.pem 2048 openssl rsa -passin pass:1234 -in /ossl/private.pem -outform PEM -pubout -out /ossl/public.pem openssl rsa -passin pass:1234 -in /ossl/private.pem -out /ossl/private_unencrypted.pem -outform PEM ``` # Progress This project is still and will be under active development with intermediate production releases. It can either be used as a standalone database or as a participating node in Aquila Network. Please note, [Aquila Port](https://github.com/Aquila-Network/specs/blob/main/README.md#aquila-port) (peer-peer network layer for Aquila DB nodes) is also a work in progress. Currently, you need to deploy your custom models to feed vector embeddings to Aquila DB, until [Aquila Hub](https://github.com/Aquila-Network/specs/blob/main/README.md#aquila-hub) developments get started. # Contribute We have [prepared a document](https://docs.google.com/document/d/1bT2_9FQIxQpx_rdYbkTukn_DJRi_haVK_ixTf8uTaDE/edit?usp=sharing) to get anyone interested to contribute, immediately started with Aquila DB. Here is our high-level [release roadmap](https://user-images.githubusercontent.com/19545678/62313851-5af82880-b4af-11e9-84f6-21e24bf46e8a.png). # Learn We have started meeting developers and do small talks on Aquila DB. Here are the slides that we use on those occasions: http://bit.ly/AquilaDB-slides **Video:** [introduction to Neural Information retrieval with AquilaDB](http://www.youtube.com/watch?v=-VYpjpLXU5Q) As of current AquilaDB release features, you can build **[Neural Information Retrieval](https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf)** applications out of the box without any external dependencies. Here are some useful links to learn more about it and start building: * Microsoft published a paper and youtube video on this to onboard anyone interested: * paper: https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf * video: https://www.youtube.com/watch?v=g1Pgo5yTIKg * Embeddings for Everything: Search in the Neural Network Era: https://www.youtube.com/watch?v=JGHVJXP9NHw * Autoencoders are one such deep learning algorithms that will help you to build semantic vectors - foundation for Neural Information retrieval. Here are some links to Autoencoders based IR: * go to chapter 15 in this link: https://www.cs.toronto.edu/~hinton/coursera_lectures.html * https://www.coursera.org/lecture/ml-foundations/examples-of-document-retrieval-in-action-CW25H * https://www.coursera.org/lecture/intro-to-deep-learning/autoencoders-101-QqBOa * Note that, the idea of information retrieval applies not only to text data but for any data. All you need to do is, encode any source datatype to a dense vector with deep neural networks.

Our Sponsors


> email us to sponsor this project [adbadmin@protonmail.ch](mailto:adbadmin@protonmail.ch).

# Citing Aquila DB If you use Aquila DB in an academic paper, we would 😍 to be cited. Here are the two ways of citing Aquila DB: ``` \footnote{https://github.com/Aquila-Network/AquilaDB} ``` ``` @misc{AquilaNetwork2019AquilaDB, title={AquilaDB: Neural Search Engine}, author={Jubin Jose, Nibin Peter}, howpublished={\url{https://github.com/Aquila-Network/AquilaDB}}, year={2019} } ``` # License Apache License 2.0 [license file](https://github.com/Aquila-Network/AquilaDB/blob/master/LICENSE) created by ❤️ with a-mma (a_മ്മ) ================================================ FILE: aquila/encoder/CODE_OF_CONDUCT.md ================================================ # AquilaDB Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at humans@aquiladb.xyz. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq ================================================ FILE: aquila/encoder/Dockerfile ================================================ # start a new build stage FROM ubuntu:latest as builder # set work directory ENV ROOT_DIR /home/root WORKDIR $ROOT_DIR # preperations ENV PATH="$ROOT_DIR/env/bin:$PATH" SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt update && apt install -y git nano python3.8 python3-pip libssl-dev && \ pip3 install virtualenv RUN cd $ROOT_DIR && \ mkdir -p ahub && \ cd ahub && \ git clone https://github.com/Aquila-Network/AquilaHub.git . && \ virtualenv $ROOT_DIR/env && \ source $ROOT_DIR/env/bin/activate && \ cd src && pip3 install -r requirements.txt RUN mkdir -p /ossl/ && \ openssl genrsa -passout pass:1234 -des3 -out /ossl/private.pem 2048 && \ openssl rsa -passin pass:1234 -in /ossl/private.pem -outform PEM -pubout -out /ossl/public.pem && \ openssl rsa -passin pass:1234 -in /ossl/private.pem -out /ossl/private_unencrypted.pem -outform PEM # install and start demon RUN mkdir -p /data && \ printf "#!/bin/bash\nsource env/bin/activate && cd ahub/src && \ python3 index.py" > /bin/init.sh && chmod +x /bin/init.sh # expose port EXPOSE 5002 ENTRYPOINT ["init.sh"] ================================================ FILE: aquila/encoder/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: aquila/encoder/README.md ================================================
Aquila Network Logo

Aquila Hub

Load and serve Neural Encoder Models


Load and serve ML models to compress data into latent vectors. To be used with Aquila DB. # Technology Aquila Hub automates the process of encoding information with the help of ML models. Here is where Aquila Hub fits in the entire ecosystem:
Aquila Hub Architecture
# Install ### Debian Run `curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaHub/main/install.sh | /bin/bash -s -- -d 1 `. ### Docker **You need docker installed in your system** Build image (one time process): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaHub/main/Dockerfile -t aquilahub:local` Run image (to deploy Aquila DB): `docker run -p 5002:5002 -d aquilahub:local` ================================================ FILE: aquila/encoder/authentication.py ================================================ from utils import cryptops import os pub_key = cryptops.read_public_key(os.environ["AUTH_KEY_FILE"]) def check (json_data, signature): return cryptops.verify_signature(json_data, pub_key, signature) ================================================ FILE: aquila/encoder/config.yml ================================================ auth: pubkey: "/ossl/public.pem" ipfs: gateway: "http://127.0.0.1:8080" ================================================ FILE: aquila/encoder/encoder.py ================================================ import logging import fasttext from utils import downloader import hashlib import base58 import json from sentence_transformers import SentenceTransformer import os # define constants MODEL_FASTTEXT = "ftxt" MODEL_S_TRANSFORMER = "strn" PREDICT_BATCH_SIZE = 1000 # Maintain a model directory data_dir = os.environ["DATA_STORE_LOCATION"] model_dir = data_dir + "models/" model_dict = {} def get_url (schema): """ Get model url from a schema """ if schema.get("encoder") != None: return schema["encoder"] else: return None def get_url_hash (url): hash_ = hashlib.sha256(url.encode('utf-8')) b58c_ = base58.b58encode(hash_.digest()) return b58c_.decode('utf-8') def download_model (url, directory, file_name): """ Download a model from a URL """ # handle fasttext models from url or IPFS if url.split(":")[0] == MODEL_FASTTEXT: url = ":".join(url.split(":")[1:]) if url.split(":")[0] == "http" or url.split(":")[0] == "https": return MODEL_FASTTEXT, downloader.http_download(url, directory, file_name+".bin") elif url.split(":")[0] == "ipfs": return MODEL_FASTTEXT, downloader.ipfs_download(url, directory, file_name+".bin") elif url.split(":")[0] == MODEL_S_TRANSFORMER: url = ":".join(url.split(":")[1:]) return MODEL_S_TRANSFORMER, url else: logging.error("Invalid encoder specified in schema definition.") return None, "" def memload_model (model_type, model_filename): """ Load a model from disk """ if model_type == MODEL_FASTTEXT: if model_filename: logging.debug("loading fasttext model into memory..") return model_type, fasttext.load_model(model_filename) else: return None, None elif model_type == MODEL_S_TRANSFORMER: if model_filename: logging.debug("loading STransformer model into memory..") return model_type, SentenceTransformer(model_filename) else: return None, None else: return None, None class EncodeRequest (): def __init__(self, id_in, text_in): self.id = id_in self.text = text_in class Encoder (): def __init__(self, encoder_name_in, request_queue_in): self.encoder_name = get_url_hash(encoder_name_in) # to handle requests self.request_queue = request_queue_in self.request_id_counter = 0 self.request_id_counter_max = 10000 # to handle responses self.response_queue = [None] * self.request_id_counter_max def __del__(self): logging.debug("killed encoder for database") def count_request_id (self): ret_ = self.request_id_counter self.request_id_counter = (self.request_id_counter + 1) % self.request_id_counter_max return ret_ def preload_model (self, json_schema, database_name): """ Download a model and load it into memory """ # prefill model & hash dictionary global model_dict try: # load model if not done already if not model_dict.get(self.encoder_name): model_type_, model_file_loc_ = download_model(get_url(json_schema), model_dir, self.encoder_name) # download success if model_file_loc_ != None: model_dict[self.encoder_name] = {} # load into memory model_dict[self.encoder_name]["type"], model_dict[self.encoder_name]["model"] = memload_model(model_type_, model_file_loc_) # memory loading failed if model_dict[self.encoder_name]["type"] == None: logging.error("Memory loading of model failed") return False else: return False if model_dict[self.encoder_name].get("model"): logging.debug("Model loaded for database: "+database_name) return True else: logging.error("Model loading failed for database: "+database_name) # reset DB - hash map del model_dict[self.encoder_name] return False else: return True except Exception as e: logging.error(e) return False async def enqueue_compress_data (self, texts): """ Add to request queue for compression """ request_ = EncodeRequest(self.count_request_id(), texts) await self.request_queue.put(request_) return request_.id async def process_queue (self): """ Load an already existing model, pop request queue, compress information, push to response queue """ request_data = [] request_metadata = [] max_batch_len = PREDICT_BATCH_SIZE # model's batching capacity # create batch from req. queue while(not self.request_queue.empty()): # get an item from queue section_ = await self.request_queue.get() request_data += section_.text request_metadata.append( (section_.id, len(section_.text)) ) # check max. batch length achieved if len(request_data) > max_batch_len: break # prefill model & hash dictionary global model_dict # model_dict[self.encoder_name] if not model_dict.get(self.encoder_name): # try dynamic loading of model try: model_dict[self.encoder_name] = {} model_dict[self.encoder_name]["type"], model_dict[self.encoder_name]["model"] = memload_model(MODEL_FASTTEXT, model_dir + self.encoder_name + ".bin") except Exception as e: logging.error("Model not pre-loaded for database.") logging.error(e) return [] result = [] try: # fasttext model prediction if model_dict[self.encoder_name]["type"] == MODEL_FASTTEXT: result = [] # fasttext doesn't take in batch; so, loop it. for line_ in request_data: result.append(model_dict[self.encoder_name]["model"].get_sentence_vector(line_).tolist()) # stransformer model prediction if model_dict[self.encoder_name]["type"] == MODEL_S_TRANSFORMER: result = model_dict[self.encoder_name]["model"].encode(request_data).tolist() except Exception as e: logging.error(e) logging.error("Model prediction error for database.") # add results to response queue self.response_queue[request_metadata[0][0]] = result[0:request_metadata[0][1]] old_metadata = request_metadata[0] for metadata_ in request_metadata[1:]: self.response_queue[metadata_[0]] = result[old_metadata[1]:old_metadata[1]+metadata_[1]] old_metadata = metadata_ ================================================ FILE: aquila/encoder/index.py ================================================ import logging from quart import Quart from quart import request from functools import wraps import asyncio from utils import config import authentication import manager as man_ app = Quart(__name__, instance_relative_config=True) # Server starter def quartserver (): """ start server """ app.run(host='0.0.0.0', port=5002, debug=False) # Add authentication def authenticate (): def decorator (f): @wraps(f) async def wrapper (*args, **kwargs): params = await extract_request_params(request) if not params or not "data" in params or not "signature" in params: return "Unauthorised access", 401 if not authentication.check(params["data"], params["signature"]): return "Unauthorised access", 401 return await f(*args, **kwargs) return wrapper return decorator async def extract_request_params (request): if not request.is_json: logging.error("Cannot parse request parameters") # request is invalid return {} # Extract JSON data data_ = await request.get_json() return data_ @app.route("/", methods=['GET']) def info (): """ Check server status """ # Build response return { "success": True, "message": "AquilaHub is running healthy" }, 200 @app.route("/prepare", methods=['POST']) @authenticate() async def prepare_model (): """ Preload and prepare model from schema definition """ # get parameters params = (await extract_request_params(request)).get("data") if not params: # Build error response return { "success": False, "message": "Invalid parameters" }, 400 if "schema" in params: database_name = app.manager.preload_model(params.get("schema")) # Build response if database_name: return { "success": True, "databaseName": database_name }, 200 else: return { "success": False, "message": "Invalid schema definition" }, 400 else: return { "success": False, "message": "Invalid parameters" }, 400 @app.route("/compress", methods=['POST']) async def compress_data (): """ generate embeddings for an input data """ # get parameters params = (await extract_request_params(request)).get("data") if not params: # Build error response return { "success": False, "message": "Invalid parameters" }, 400 if "text" in params and "databaseName" in params: vectors = await app.manager.compress_data(params.get("databaseName"), params.get("text")) # Build response if vectors: return { "success": True, "vectors": vectors }, 200 else: return { "success": False, "message": "Database not found" }, 400 else: return { "success": False, "message": "Invalid parameters" }, 400 @app.before_serving async def init_variables(): app.manager = man_.Manager() # prepare HUB from backup try: app.manager.prepare_hub() except Exception as e: logging.error("Backup restore failed") logging.error(e) # initialize background task controller app.manager.bg_task_active = True @app.before_serving async def init_tasks(): # initialize background task app.manager.background_task = asyncio.ensure_future(app.manager.background_task()) # app.add_background_task(background_task) @app.after_serving async def shutdown(): # shutdown background task app.manager.bg_task_active = False app.manager.background_task.cancel() if __name__ == "__main__": quartserver() ================================================ FILE: aquila/encoder/ipfs.service ================================================ [Unit] Description=IPFS daemon After=network.target [Service] ### custom ipfs datastore location # Environment=IPFS_PATH=/path/to/your/ipfs/datastore ExecStart=/usr/local/bin/ipfs daemon Restart=on-failure [Install] WantedBy=default.target ================================================ FILE: aquila/encoder/manager.py ================================================ import logging logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) import asyncio import os import json import copy from utils import CID, schema import encoder as enc_ SLEEP_PROTECTION = 0.0001 def get_database_name (schema_in): """ Get databse name from schema """ database_name = None try: schema_def = schema.generate_schema(schema_in) database_name = CID.doc2CID(schema_def) except Exception as e: logging.error(e) return database_name def get_encoder_name (schema_in): """ Get encoder name from schema """ encoder_name = None try: schema_def = schema.generate_schema(schema_in) encoder_name = schema_def["encoder"] except Exception as e: logging.error(e) return encoder_name class Manager (): def __init__ (self): # to track all database - encoder mappings self.db_to_encoders_map = {} self.encoders_to_obj_map = {} def __del__ (self): logging.debug("Killed manager") def persist_db (self, json_schema, database_name, persist=True): # do nothing if not persist: logging.debug("Skipping schema persist to Disk") return database_name # create backup directory location = os.environ["DATA_STORE_LOCATION"]+"hub_backup/" try: if not os.path.exists(location): # create backup directory os.mkdir(location) except Exception as e: logging.error("Backup directory creation failed") logging.error(e) return None # store schema, replacing the old one try: with open(location+database_name, "w") as f_: json.dump(json_schema, f_) except Exception as e: logging.error("Schema JSON writing failed") logging.error(e) return None return database_name # initial preperation of Aquila HUB from backup def prepare_hub (self): # load schema files location = os.environ["DATA_STORE_LOCATION"]+"hub_backup/" for database_name in os.listdir(location): with open(location+database_name, "r") as schema__: schema_ = json.load(schema__) database_name_ = self.preload_model(schema_, persist=False) if database_name_ == None: logging.debug("Model preloading failed for database: "+database_name) elif database_name_ == database_name: logging.debug("Model preloaded for database: "+database_name) else: logging.debug("Model missmatch for database "+database_name+", schema backup is modified. Hope you know what you're doing.") def preload_model (self, json_schema, persist=True): """ Download a model and load it into memory """ # create a schema copy schema_copy = copy.deepcopy(json_schema) # parse schema # NOTE: get_database_name makes modifications to "json_schema" variable # so, beware of it's subsequent usage. If you want to use the original schema, # use "schema_copy" variable instead database_name = get_database_name(json_schema) encoder_name = get_encoder_name(json_schema) # load model for database if database_name: # database already not created? if not self.db_to_encoders_map.get(database_name): self.db_to_encoders_map[database_name] = encoder_name # encoder already not created? if not self.encoders_to_obj_map.get(encoder_name): self.encoders_to_obj_map[encoder_name] = enc_.Encoder(encoder_name, asyncio.Queue()) # encoder already loaded? if self.encoders_to_obj_map[encoder_name].preload_model(json_schema, database_name): return self.persist_db(schema_copy, database_name, persist) else: # reset all, don't create DB & encoder self.encoders_to_obj_map[encoder_name] = None self.db_to_encoders_map[database_name] = None return None else: return self.persist_db(schema_copy, database_name, persist) else: return self.persist_db(schema_copy, database_name, persist) else: return None async def compress_data (self, database_name, texts): """ Load an already existing database """ if self.db_to_encoders_map.get(database_name): encoder_name = self.db_to_encoders_map[database_name] if self.encoders_to_obj_map.get(encoder_name): response_ = None # add compression request to queue, get request id req_id = await self.encoders_to_obj_map[encoder_name].enqueue_compress_data(texts) # wait until request id is processed while (True): # response available yet? response_ = self.encoders_to_obj_map[encoder_name].response_queue[req_id] if response_ != None: # response available; take it, reset queue & break waiting self.encoders_to_obj_map[encoder_name].response_queue[req_id] = None break # sleep for a while await asyncio.sleep(SLEEP_PROTECTION) return response_ else: return None else: return None # define background task to process request queue for each database object async def background_task(self): logging.debug("===== Background task INIT =====") while self.bg_task_active: # for each database object for key_ in self.encoders_to_obj_map: # any request available in queue? if self.encoders_to_obj_map[key_].request_queue.empty(): continue # process request await self.encoders_to_obj_map[key_].process_queue() await asyncio.sleep(SLEEP_PROTECTION) ================================================ FILE: aquila/encoder/requirements.txt ================================================ aiofiles==0.8.0 base58==2.1.1 blinker==1.4 bson==0.5.10 certifi==2021.10.8 chardet==4.0.0 charset-normalizer==2.0.10 click==8.0.3 fastjsonschema==2.15.3 fasttext==0.9.2 filelock==3.4.2 h11==0.12.0 h2==4.1.0 hpack==4.0.0 huggingface-hub==0.4.0 hypercorn==0.13.2 hyperframe==6.0.1 idna==3.3 itsdangerous==2.0.1 Jinja2==3.0.3 joblib==1.1.0 MarkupSafe==2.0.1 nltk==3.6.7 numpy==1.22.0 packaging==21.3 Pillow==9.0.0 priority==2.0.0 pybind11==2.9.0 pycryptodome==3.12.0 pyparsing==3.0.6 python-dateutil==2.8.2 PyYAML==6.0 quart==0.16.2 regex==2021.11.10 requests==2.27.1 sacremoses==0.0.47 scikit-learn==1.0.2 scipy==1.7.3 sentence-transformers==2.1.0 sentencepiece==0.1.96 six==1.16.0 threadpoolctl==3.0.0 tokenizers==0.10.3 toml==0.10.2 torch==1.10.1 torchvision==0.11.2 tqdm==4.62.3 transformers==4.15.0 typing-extensions==4.0.1 urllib3==1.26.8 Werkzeug==2.0.2 wsproto==1.0.0 ================================================ FILE: aquila/encoder/run_tests.sh ================================================ # rm -r /data/* python3 -m unittest test.apis.hub_fns -v ================================================ FILE: aquila/encoder/test/__init__.py ================================================ ================================================ FILE: aquila/encoder/test/apis/hub_fns.py ================================================ import unittest from utils import CID, schema from Crypto.Hash import SHA384 from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 import base58 import requests from requests.structures import CaseInsensitiveDict import json import bson host = "127.0.0.1:5002" # load private key with open("private_unencrypted.pem", "r") as pkf: k = pkf.read() priv_key = RSA.import_key(k) class TestAuth (unittest.TestCase): # Preload a model def test_1_auth_preload_http (self): schema_def_ = { "description": "this is my database", "unique": "r8and0mseEd905", "encoder": "ftxt:https://ftxt-models.s3.us-east-2.amazonaws.com/ftxt_base_min.bin", "codelen": 25, "metadata": {} } # generate schema schema_def = schema.generate_schema(schema_def_) database_name = CID.doc2CID(schema_def) # 1. test preperation data_ = { "schema": schema_def_ } data_bson = bson.dumps(data_) # generate hash hash = SHA384.new() hash.update(data_bson) # Sign with pvt key signer = pkcs1_15.new(priv_key) signature = signer.sign(hash) signature = base58.b58encode(signature).decode("utf-8") url = "http://"+host+"/prepare" headers = CaseInsensitiveDict() headers["Content-Type"] = "application/json" data = { "data": data_, "signature": signature } data = json.dumps(data) resp = requests.post(url, headers=headers, data=data) database_name_ = resp.json()["databaseName"] # check databases are the same self.assertEqual(database_name, database_name_, "DB name doesn't match") # 2. test compression data_ = {"databaseName": database_name_, "text": ["data one", "data two"]} url = "http://"+host+"/compress" headers = CaseInsensitiveDict() headers["Content-Type"] = "application/json" data = { "data": data_ } data = json.dumps(data) resp = requests.post(url, headers=headers, data=data) # # check returned array is valid self.assertEqual(len(resp.json()["vectors"]), len(data_["text"]), "Compressed items doesn't match query") self.assertEqual(len(resp.json()["vectors"][0]), schema_def_["codelen"], "Compressed code length doesn't match") # Preload a model def test_2_auth_preload_ipfs (self): schema_def_ = { "description": "this is my database", "unique": "r8and0mseEd905", "encoder": "ftxt:ipfs://QmT9CECrTwUhAPw6VHgxcLciH4EBepkXJmSB9y2rchsVQz", "codelen": 25, "metadata": {} } # generate schema schema_def = schema.generate_schema(schema_def_) database_name = CID.doc2CID(schema_def) # 1. test preperation data_ = { "schema": schema_def_ } data_bson = bson.dumps(data_) # generate hash hash = SHA384.new() hash.update(data_bson) # Sign with pvt key signer = pkcs1_15.new(priv_key) signature = signer.sign(hash) signature = base58.b58encode(signature).decode("utf-8") url = "http://"+host+"/prepare" headers = CaseInsensitiveDict() headers["Content-Type"] = "application/json" data = { "data": data_, "signature": signature } data = json.dumps(data) resp = requests.post(url, headers=headers, data=data) database_name_ = resp.json()["databaseName"] # check databases are the same self.assertEqual(database_name, database_name_, "DB name doesn't match") # 2. test compression data_ = {"databaseName": data_["databaseName"], "text": ["data one", "data two"]} url = "http://"+host+"/compress" headers = CaseInsensitiveDict() headers["Content-Type"] = "application/json" data = { "data": data_ } data = json.dumps(data) resp = requests.post(url, headers=headers, data=data) # check returned array is valid self.assertEqual(len(resp.json()["vectors"]), len(data_["text"]), "Compressed items doesn't match query") self.assertEqual(len(resp.json()["vectors"][0]), schema_def_["codelen"], "Compressed code length doesn't match") if __name__ == '__main__': unittest.main() ================================================ FILE: aquila/encoder/test.py ================================================ from aquilapy import Wallet, DB, Hub import numpy as np import time import multiprocessing as mp # Create a wallet instance from private key wallet = Wallet("/home/iamjbn/aquilax/ossl/private_unencrypted.pem") host = "http://127.0.0.1" # Connect to Aquila Hub instance hub = Hub(host, "5002", wallet) # Schema definition to be used schema_def = { "description": "AquilaX-CE default user index", "unique": "user_id1", "encoder": "ftxt:http://0.0.0.0:2000/cc.en.300.bin", "codelen": 300, "metadata": { "url": "string", "text": "string" } } def run_test (index_in): print('index_in:', index_in) # Craete a database with the schema definition provided db_name_ = hub.create_database(schema_def) # Generate encodings texts = ["Sure, technically these are processes, and this program should really be called a process spawning manager, but this is only due to the way that BASH works when it forks using the ampersand, it uses the fork() or perhaps clone() system call which clones into a separate memory space, rather than something like pthread_create() which would share memory. If BASH supported the latter, each sequence of execution would operate just the same and could be termed to be traditional threads whilst gaining a more efficient memory footprint.", "Sure, technically these are processes, and this program should really be called a process spawning manager, but this is only due to the way that BASH works when it forks using the ampersand, it uses the fork() or perhaps clone() system call which clones into a separate memory space, rather than something like pthread_create() which would share memory. If BASH supported the latter, each sequence of execution would operate just the same and could be termed to be traditional threads whilst gaining a more efficient memory footprint.", "Sure, technically these are processes, and this program should really be called a process spawning manager, but this is only due to the way that BASH works when it forks using the ampersand, it uses the fork() or perhaps clone() system call which clones into a separate memory space, rather than something like pthread_create() which would share memory. If BASH supported the latter, each sequence of execution would operate just the same and could be termed to be traditional threads whilst gaining a more efficient memory footprint.", "Sure, technically these are processes, and this program should really be called a process spawning manager, but this is only due to the way that BASH works when it forks using the ampersand, it uses the fork() or perhaps clone() system call which clones into a separate memory space, rather than something like pthread_create() which would share memory. If BASH supported the latter, each sequence of execution would operate just the same and could be termed to be traditional threads whilst gaining a more efficient memory footprint."] compression = hub.compress_documents(db_name_, texts) print(len(compression)==len(texts)) # test concurrency def test_concurrency_thread (counter_upto): start = time.time() print("starting thread") pool = mp.Pool(500) #mp.cpu_count()) pool.map(run_test, range(counter_upto)) pool.close() pool.join() end = time.time() print(end-start, counter_upto) start = end test_concurrency_thread(10000) # test model downloads from different sources schema_def["encoder"] = "ftxt:http://0.0.0.0:2000/cc.en.300.bin" schema_def["codelen"] = 300 print(schema_def) db_name_ = hub.create_database(schema_def) print(db_name_) # test model downloads from different sources schema_def["encoder"] = "ftxt:ipfs://QmY2FFRuW4xVeCDkwgwkWcq1aHaKFjfbEHPXjEEuQYax4P" schema_def["codelen"] = 300 db_name_ = hub.create_database(schema_def) print(db_name_) ================================================ FILE: aquila/encoder/utils/CID.py ================================================ import logging import bson import hashlib import base58 def doc2CID (inp): try: # JSON OBJ to BSON encode bson_ = bson.dumps(inp) # SHA-256 Double Hashing hash_ = hashlib.sha256(bson_) hash_ = hashlib.sha256(hash_.digest()) # Convert to Base-58 string b58c_ = base58.b58encode(hash_.digest()) return b58c_.decode('utf-8') except Exception as e: logging.debug(e) return None def doc2bson (inp): try: # JSON OBJ to BSON encode bson_ = bson.dumps(inp) return bson_ except Exception as e: logging.debug(e) return None def bson2doc (inp): try: # BSON to JSON OBJ json_ = bson.loads(inp) return json_ except Exception as e: logging.debug(e) return None ================================================ FILE: aquila/encoder/utils/config.py ================================================ import yaml import os os.environ["DATA_STORE_LOCATION"] = "/data/" with open("config.yml", "r") as stream: DB_config = yaml.safe_load(stream) if "AUTH_KEY_FILE" not in os.environ: os.environ["AUTH_KEY_FILE"] = str(DB_config["auth"]["pubkey"]) if "IPFS_GATEWAY" not in os.environ: os.environ["IPFS_GATEWAY"] = str(DB_config["ipfs"]["gateway"]) ================================================ FILE: aquila/encoder/utils/cryptops.py ================================================ import logging from Crypto.Hash import SHA384 from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 import base58 import chardet import bson def read_public_key (location): # disk read public key with open(location, "r") as pkf: k = pkf.read() pub_key = RSA.import_key(k) return pub_key def verify_signature (json_data, pub_key, signature): ret = True binary_data = bson.dumps(json_data) # generate hash hash = SHA384.new() hash.update(binary_data) signature = base58.b58decode(signature) # Verify with pub key verifier = pkcs1_15.new(pub_key) try: verifier.verify(hash, signature) except Exception as e: logging.debug(e) ret = False return ret ================================================ FILE: aquila/encoder/utils/downloader.py ================================================ import logging import os import time import requests from tqdm import tqdm def download_large_file (url, location, method="get"): r = None if method == "get": r = requests.get(url, stream=True) if method == "post": r = requests.post(url, stream=True) if r != None: r.raise_for_status() total_size_in_bytes= int(r.headers.get('content-length', 0)) block_size = 1024*5 progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True) with open(location, 'wb') as f: for chunk in r.iter_content(chunk_size=block_size): progress_bar.update(len(chunk)) f.write(chunk) r.close() progress_bar.close() if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes: logging.error("Download failed.") os.remove(location) return False return True def http_download (url, directory, file_name): """ Download model via http """ try: # create models dir, if not exists if not os.path.exists(directory): os.makedirs(directory) original_bin_file_name = url.split("/")[-1] # check if model already exists if not os.path.exists(directory+original_bin_file_name): # download file logging.debug("Downloading model..") # download from given url status = download_large_file(url, directory+original_bin_file_name) # failed?? if not status: return None if not os.path.exists(directory+file_name): # copy and rename model logging.debug("Copy model..") os.symlink(directory+original_bin_file_name, directory+file_name) except Exception as e: logging.error(e) return None return directory+file_name def ipfs_download (url, directory, file_name): """ Download model via IPFS """ try: # create models dir, if not exists if not os.path.exists(directory): os.makedirs(directory) IPFS_CID = url.split("ipfs://")[1] original_bin_file_name = IPFS_CID # check if model already exists if not os.path.exists(directory+original_bin_file_name): # download file logging.debug("Downloading model..") logging.debug("Connecting to local IPFS demon..") # download from IPFS local API url_ = os.environ["IPFS_GATEWAY"]+"/ipfs/"+IPFS_CID status = download_large_file(url_, directory+original_bin_file_name) # failed?? if not status: return None if not os.path.exists(directory+file_name): # copy and rename model logging.debug("Copy model..") os.symlink(directory+original_bin_file_name, directory+file_name) except Exception as e: logging.error(e) return None return directory+file_name ================================================ FILE: aquila/encoder/utils/schema.py ================================================ import logging import fastjsonschema def generate_schema (template): # get all keys from template schema metadata_templ = template.get("metadata") if metadata_templ: del template["metadata"] else: metadata_templ = {} keys_ = list(template.keys()) # sort keys keys_.sort() # validate required keys req_k = ["description", "encoder", "unique"] for r_ in req_k: if r_ not in keys_: # cannot continue, invalid schema return None # generate schema in sorted keys generated_schema = { "$schema": "http://json-schema.org/schema#", "$id": "http://aquilanetwork.com/schema/id/v1", "title": "Schema", "description": "", "encoder": "", "unique": "", "type": "object", "properties": { "code": { "description": "Encoded data", "type": "array", "items": { "type": "number" } }, "metadata": { "$ref": "#/definitions/metadata" } }, "definitions": { "metadata": { "description": "User defined metadata", "type": "object", "properties": {}, "required": [] } }, "required": ["code", "metadata"] } metadata = {} metadata_types = ["number", "string"] for key_ in keys_: if key_ in req_k: # fill in root level keys generated_schema[key_] = template[key_] elif key_ == "codelen": generated_schema["properties"]["code"]["maxItems"] = template[key_] else: # fill in root level keys :: extra info that we don't care generated_schema[key_] = template[key_] for key_ in metadata_templ.keys(): # check type is predefined if metadata_templ[key_] not in metadata_types: return None # fill in metadata keys metadata[key_] = { "type": metadata_templ[key_] } generated_schema["definitions"]["metadata"]["properties"] = metadata generated_schema["definitions"]["metadata"]["required"] = list(metadata.keys()) # validate values if type(generated_schema["properties"]["code"].get("maxItems")) != int: return None return generated_schema def compile (schema_def): # compile schema validator = fastjsonschema.compile(schema_def) return validator def validate_json_docs (validator, json_doc): try: # validate doc on schema :: will except on fail validator(json_doc) # validated # logging.debug("Schema validation success") return True except Exception as e: logging.error(e) return False ================================================ FILE: aquila/metricstore/.travis.yml ================================================ language: node_js services: - docker node_js: - "11" before_install: - docker build -f Dockerfile_local_build -t ammaorg/aquiladb:travis . - docker run -d -i -p 50051:50051 -t ammaorg/aquiladb:travis - docker ps -a - sudo apt-get install -y make - cd src install: - npm install - cd test script: - node test.js branches: only: - develop ================================================ FILE: aquila/metricstore/CODE_OF_CONDUCT.md ================================================ # AquilaDB Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at humans@aquiladb.xyz. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq ================================================ FILE: aquila/metricstore/DB_config.yml ================================================ docs: cswap: 10101 # minimum data required to start indexing vd: 784 # fixed vector dimension faiss: init: nlist: 1 # number of cells nprobe: 1 # number of cells that are visited to perform a search bpv: 8 # bytes per vector bpsv: 8 # bytes per sub vector annoy: init: smetric: "dot" # similarity metric to be used ntrees: 500 # no. of trees search_k: -1 # search_k is provided in runtime and affects the search performance. A larger value will give more accurate results, but will take longer time to return. queue: qlen: 100000 # length limit for quesues used sleep: 1 # thread waiting in seconds auth: pubkey: "/ossl/public.pem" ================================================ FILE: aquila/metricstore/Dockerfile ================================================ # start a new build stage FROM ubuntu:latest as builder RUN ls # set work directory ENV ROOT_DIR /home/root WORKDIR $ROOT_DIR # install aquiladb RUN apt update && apt install -y curl && \ curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/install.sh | /bin/bash # preperations ENV PATH="$ROOT_DIR/env/bin:$PATH" SHELL ["/bin/bash", "-o", "pipefail", "-c"] # install and start demon RUN mkdir -p /data && \ printf "#!/bin/bash\nsource env/bin/activate && export MINI_AQDB='active' && cd adb/src && \ python3 index.py" > /bin/init.sh && chmod +x /bin/init.sh # expose port EXPOSE 5001 ENTRYPOINT ["init.sh"] ================================================ FILE: aquila/metricstore/DockerfileBig ================================================ # start a new build stage FROM ubuntu:latest as builder # set work directory ENV ROOT_DIR /home/root WORKDIR $ROOT_DIR # install aquiladb RUN apt update && apt install -y curl && \ curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/install.sh | /bin/bash -s -- -m 0 # start a new runner stage FROM ubuntu:latest as runner # set work directory ENV ROOT_DIR /home/root WORKDIR $ROOT_DIR RUN echo "$ROOT_DIR" # copy required files from builder stage COPY --from=builder $ROOT_DIR/env $ROOT_DIR/env COPY --from=builder $ROOT_DIR/adb $ROOT_DIR/adb COPY --from=builder /ossl /ossl # preperations ENV PATH="$ROOT_DIR/env/bin:$PATH" WORKDIR $ROOT_DIR SHELL ["/bin/bash", "-o", "pipefail", "-c"] # install and start demon RUN export DEBIAN_FRONTEND=noninteractive && mkdir -p /data && apt update && \ apt install -y python3 libgomp1 libblas-dev liblapack-dev && \ printf "#!/bin/bash\nsource env/bin/activate && cd adb/src && \ python3 index.py" > /bin/init.sh && chmod +x /bin/init.sh # expose port EXPOSE 5001 ENTRYPOINT ["init.sh"] ================================================ FILE: aquila/metricstore/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: aquila/metricstore/README.md ================================================
Aquila Network Logo

Aquila DB

Easy to use Neural Search Engine


**[Aquila DB](https://github.com/Aquila-Network/AquilaDB)** is a Neural search engine. In other words, it is a database to index **Latent Vectors** generated by ML models along with **JSON Metadata** to perform **k-NN** retrieval. It is dead simple to set up, language-agnostic, and drop in addition to your Machine Learning Applications. Aquila DB, as of current features is a ready solution for Machine Learning engineers and Data scientists to build **[Neural Information Retrieval](https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf)** applications out of the box with minimal dependencies. > This project is still in alpha version & we're already using it in production to power semantic search at https://aquila.network. Wanna support this project? Yes, we love getting a **star** ⭐ and **shout-out** 🗣️ 🤗 Join [Community chat and get support: ![discord chatroom for discussions](https://www.freeiconspng.com/minicovers/flat-discord-material-like-icon--2.png)](https://discord.gg/5YP7zHS) # Who is this for * If you are working on a data science project and need to store a hell of a lot of data and retrieve similar data based on some feature vector, this will be a useful tool to you, with extra benefits a real world web application needs. * Are you dealing with a lot of images and related metadata? Want to find similar ones? You are at the right place. * If you are looking for a document database, this is not the right place for you. # Technology Aquila DB powers search features of Aquila Network. Here is where Aquila DB fits in the entire ecosystem:
Aquila DB Architecture
If you are serious and wanna dive down the rabbit hole, read our **[whitepapers](https://github.com/Aquila-Network/whitepaper)** and **[technical specifications](https://github.com/Aquila-Network/specs)** (being actively worked on). **As a side note**, everything in **[Aquila Network](https://github.com/Aquila-Network)** is defined by the specifications and a large chunk of our efforts goes into it. We also maintain quality implementations of those specifications with non-technical users in mind. This is to make sure that - Aquila Network is fully open, decentralized by design, and Fair. You can follow those specifications to implement your alternative software and still interact with the network without any restrictions. # Install ### Debian Run `curl -s -L https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/install.sh | /bin/bash -s -- -d 1 `. ### Docker **You need docker installed in your system** Build image (lite): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/Dockerfile -t aquiladb:local` Build image (big data): `docker build https://raw.githubusercontent.com/Aquila-Network/AquilaDB/master/DockerfileBig -t aquiladb:localbg` Run image (to deploy Aquila DB lite): `docker run -p 5001:5001 -d aquiladb:local` Run image (to deploy Aquila DB big): `docker run -p 5001:5001 -d aquiladb:localbg` # Client SDKs We currently have multiple client libraries in progress to abstract the communication between deployed Aquila DB and your applications. [Python](https://github.com/Aquila-Network/AquilaPy) [Node JS](https://github.com/Aquila-Network/AquilaJS) ## Where to get private key (wallet key) for client authentication When you use a client library to authenticate with AquilaDB, you might need access the same private key (wallet key) used by AquilaDB. This key is located inside `/ossl/` directory within AquilaDB docker container (in your computer if you have installed AquilaDB directly without docker). To access the keys inside your AquilaDB container, follow below steps: * identify `CONTAINER ID` for the already running `aquiladb` docker instance: `docker ps` * take a copy of private keys from docker container to your host machine: `docker cp CONTAINER_ID:/ossl/ ./` * now you will see a new directory named `ossl` at your current location. Use the keys inside it. #### tips for advanced users If your pipeline requires the private keys to be generated in advance, you can do it in your host machine and then mount it to the container's `/ossl/` directory. Run: ``` mkdir -p /ossl/ openssl genrsa -passout pass:1234 -des3 -out /ossl/private.pem 2048 openssl rsa -passin pass:1234 -in /ossl/private.pem -outform PEM -pubout -out /ossl/public.pem openssl rsa -passin pass:1234 -in /ossl/private.pem -out /ossl/private_unencrypted.pem -outform PEM ``` # Progress This project is still and will be under active development with intermediate production releases. It can either be used as a standalone database or as a participating node in Aquila Network. Please note, [Aquila Port](https://github.com/Aquila-Network/specs/blob/main/README.md#aquila-port) (peer-peer network layer for Aquila DB nodes) is also a work in progress. Currently, you need to deploy your custom models to feed vector embeddings to Aquila DB, until [Aquila Hub](https://github.com/Aquila-Network/specs/blob/main/README.md#aquila-hub) developments get started. # Contribute We have [prepared a document](https://docs.google.com/document/d/1bT2_9FQIxQpx_rdYbkTukn_DJRi_haVK_ixTf8uTaDE/edit?usp=sharing) to get anyone interested to contribute, immediately started with Aquila DB. Here is our high-level [release roadmap](https://user-images.githubusercontent.com/19545678/62313851-5af82880-b4af-11e9-84f6-21e24bf46e8a.png). # Learn We have started meeting developers and do small talks on Aquila DB. Here are the slides that we use on those occasions: http://bit.ly/AquilaDB-slides **Video:** [introduction to Neural Information retrieval with AquilaDB](http://www.youtube.com/watch?v=-VYpjpLXU5Q) As of current AquilaDB release features, you can build **[Neural Information Retrieval](https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf)** applications out of the box without any external dependencies. Here are some useful links to learn more about it and start building: * Microsoft published a paper and youtube video on this to onboard anyone interested: * paper: https://www.microsoft.com/en-us/research/uploads/prod/2017/06/INR-061-Mitra-neuralir-intro.pdf * video: https://www.youtube.com/watch?v=g1Pgo5yTIKg * Embeddings for Everything: Search in the Neural Network Era: https://www.youtube.com/watch?v=JGHVJXP9NHw * Autoencoders are one such deep learning algorithms that will help you to build semantic vectors - foundation for Neural Information retrieval. Here are some links to Autoencoders based IR: * go to chapter 15 in this link: https://www.cs.toronto.edu/~hinton/coursera_lectures.html * https://www.coursera.org/lecture/ml-foundations/examples-of-document-retrieval-in-action-CW25H * https://www.coursera.org/lecture/intro-to-deep-learning/autoencoders-101-QqBOa * Note that, the idea of information retrieval applies not only to text data but for any data. All you need to do is, encode any source datatype to a dense vector with deep neural networks.

Our Sponsors


> email us to sponsor this project [adbadmin@protonmail.ch](mailto:adbadmin@protonmail.ch).

# Citing Aquila DB If you use Aquila DB in an academic paper, we would 😍 to be cited. Here are the two ways of citing Aquila DB: ``` \footnote{https://github.com/Aquila-Network/AquilaDB} ``` ``` @misc{AquilaNetwork2019AquilaDB, title={AquilaDB: Neural Search Engine}, author={Jubin Jose, Nibin Peter}, howpublished={\url{https://github.com/Aquila-Network/AquilaDB}}, year={2019} } ``` # License Apache License 2.0 [license file](https://github.com/Aquila-Network/AquilaDB/blob/master/LICENSE) created by ❤️ with a-mma (a_മ്മ) ================================================ FILE: aquila/metricstore/authentication.py ================================================ from utils import cryptops import os pub_key = cryptops.read_public_key(os.environ["AUTH_KEY_FILE"]) def check (json_data, signature): return cryptops.verify_signature(json_data, pub_key, signature) ================================================ FILE: aquila/metricstore/index.py ================================================ import logging from flask import Flask, request from flask_cors import CORS from flask import jsonify from functools import wraps from utils import config import authentication import router app = Flask(__name__, instance_relative_config=True) # Enable CORS CORS(app) # preload databases router.preload_databases() # Add authentication def authenticate (): def decorator (f): @wraps(f) def wrapper (*args, **kwargs): params = extract_request_params(request) if not params or not "data" in params or not "signature" in params: return "Unauthorised access", 401 if not authentication.check(params["data"], params["signature"]): return "Unauthorised access", 401 return f(*args, **kwargs) return wrapper return decorator def extract_request_params (request): if not request.is_json: logging.error("Cannot parse request parameters") # request is invalid return None # Extract JSON data data_ = request.get_json() return data_ @app.route("/", methods=['GET']) def info (): """ Check server status """ # Build response return { "success": True, "message": "AquilaDB is running healthy" }, 200 @app.route("/db/create", methods=['POST']) @authenticate() def db_create (): """ Create database from schema definition """ # get parameters params = extract_request_params(request)["data"] if not params: # Build error response return { "success": False, "message": "Invalid parameters" }, 400 if "schema" in params: database_name = router.create_database(params.get("schema")) # Build response if database_name: return { "success": True, "database_name": database_name }, 200 else: return { "success": False, "message": "Invalid schema definition" }, 400 @app.route("/db/doc/insert", methods=['POST']) @authenticate() def doc_insert (): """ insert documents """ # get parameters params = extract_request_params(request)["data"] if not params: # Build error response return { "success": False, "message": "Invalid parameters" }, 400 if "docs" in params and "database_name" in params: cids = router.insert_docs(params.get("docs"), params.get("database_name")) # Build response return { "success": True, "ids": cids }, 200 @app.route("/db/doc/delete", methods=['POST']) @authenticate() def doc_delete (): """ delete documents by cid """ # get parameters params = extract_request_params(request)["data"] if not params: # Build error response return { "success": False, "message": "Invalid parameters" }, 400 if "ids" in params and "database_name" in params: ids = router.delete_docs(params.get("ids"), params.get("database_name")) # Build response return { "success": True, "ids": ids }, 200 @app.route("/db/search", methods=['GET']) @authenticate() def db_search (): """ search documents """ # get parameters params = extract_request_params(request)["data"] if not params: # Build error response return { "success": False, "message": "Invalid parameters" }, 400 if "matrix" in params and "k" in params and "database_name" in params: docs, dists = router.search(params.get("matrix"), params.get("k"), None, params.get("database_name")) # Build response return { "success": True, "docs": docs, "dists": dists }, 200 def flaskserver (): """ start server """ app.run(host='0.0.0.0', port=5001, debug=False) if __name__ == "__main__": flaskserver() ================================================ FILE: aquila/metricstore/install.sh ================================================ #!/bin/bash -e export DEBIAN_FRONTEND=noninteractive apt update export USER=$(whoami) export ROOT_DIR=/home/$USER mkdir -p /data/ mkdir -p $ROOT_DIR cd $ROOT_DIR mini=1 gpu=0 test=0 demon=0 while getopts m:g:t:d: flag do case "${flag}" in m) mini=${OPTARG};; g) gpu=${OPTARG};; t) test=${OPTARG};; d) demon=${OPTARG};; esac done if [[ $mini -eq 1 ]]; # if minimal install enabled then echo "Aquila DB minimal install will be done by default. \ Minimal install is recommended for personal use - \ install process is fast and lightweight. If you are planning to deal \ with big-data, disable minimal install with '-m 0' argument" fi # system packs install apt install -y git wget nano python3 python3-pip libssl-dev if [[ $mini -eq 0 ]]; # if minimal install disabled then apt install -y libblas-dev liblapack-dev swig fi # setup venv pip3 install virtualenv virtualenv $ROOT_DIR/env source $ROOT_DIR/env/bin/activate # install python packages pip3 install numpy pycryptodome base58 chardet Flask requests flask_cors PyYAML bson fastjsonschema annoy plyvel if [[ $mini -eq 0 ]]; # if minimal install disabled then # install cmake apt purge --auto-remove cmake version=3.19 build=1 mkdir -p $ROOT_DIR/temp cd $ROOT_DIR/temp wget https://cmake.org/files/v$version/cmake-$version.$build.tar.gz tar -xzvf cmake-$version.$build.tar.gz cd cmake-$version.$build/ ./bootstrap make -j$(nproc) make install cmake --version # install faiss cd $ROOT_DIR mkdir -p faiss cd faiss git clone https://github.com/facebookresearch/faiss.git . if [[ $gpu -eq 0 ]]; # if gpu not enabled then echo "build FAISS without GPU" cmake -DFAISS_ENABLE_GPU=OFF -B build . else echo "build FAISS with GPU" cmake -B build . fi make -C build # For the Python interface: cd build/faiss/python python setup.py install cp -r $ROOT_DIR/faiss/build/faiss/python/ $ROOT_DIR/env/lib/python3.8/site-packages/faiss fi # clone & test AquilaDB cd $ROOT_DIR mkdir -p adb cd adb git clone https://github.com/Aquila-Network/AquilaDB.git . mkdir -p /ossl/ openssl genrsa -passout pass:1234 -des3 -out /ossl/private.pem 2048 openssl rsa -passin pass:1234 -in /ossl/private.pem -outform PEM -pubout -out /ossl/public.pem openssl rsa -passin pass:1234 -in /ossl/private.pem -out /ossl/private_unencrypted.pem -outform PEM cd $ROOT_DIR/adb/src if [[ $test -eq 1 ]]; # if tests enabled then chmod +x run_tests.sh ./run_tests.sh else echo "Not running tests" fi echo "===================================" echo " AquilaDB setup complete. " echo "===================================" if [[ $demon -eq 1 ]]; # if demon run enabled then # install node apt install -y nodejs npm # install pm2 npm i pm2 -g # start server pm2 start index.py # Keep logs alive pm2 logs -f fi ================================================ FILE: aquila/metricstore/manager.py ================================================ import logging from utils import CID, schema from vec_index import hannoy import os is_mini_instance = os.environ["MINI_AQDB"] if is_mini_instance == "inactive": from vec_index import hfaiss import plyvel import numpy as np import json import random import threading import queue import time import pickle INDEX_LABEL = ["annoy", "faiss"] STORE_LOCATION = os.environ["DATA_STORE_LOCATION"] TRAIN_DAT_LEN = int(os.environ["MIN_SAWP_COUNT"]) PROCESS_TIMEOUT = int(os.environ["THREAD_SLEEP"]) MAX_Q_LEN = int(os.environ["FIXED_Q_LEN"]) def byt (inp): return bytes(str(inp), 'ascii') class VecManager: def __init__ (self, json_schema): # get database name from schema CID database_name = CID.doc2CID(json_schema) # keep database name self.database_name = database_name # set DB disk location self.DB_disk_location = STORE_LOCATION + database_name # create data directory for database if not os.path.exists(self.DB_disk_location): os.makedirs(self.DB_disk_location) # keep schema in store location with open(self.DB_disk_location + '/schema.json', 'w') as oschema: json.dump(json_schema, oschema) # get vector index self.active_index = INDEX_LABEL[0] self.index = self.get_index(self.DB_disk_location) # Create KV store instance self.KV_store = plyvel.DB(self.DB_disk_location + "/kv.db", create_if_missing=True) if self.KV_store.get(byt(-1)) == None: self.KV_store.put(byt(-1), byt(0)) # Training data holder self.training_data = [] self.TD_location = self.DB_disk_location + "/TD" # Try loading training data self.load_TD_from_disk() # spawn worker thread self.q_maxsize = MAX_Q_LEN self.process_flag = True self.process_timeout_sec = PROCESS_TIMEOUT self.spawn() def __del__(self): self.process_flag = False if self.process_thread: self.process_thread.join() logging.debug("Thread stopped") # close level database connection self.KV_store.close() def add_vectors (self, documents): # add to KV store next_index = int(self.KV_store.get(byt(-1))) # check if it is ready to swap index if is_mini_instance == "inactive" and next_index > TRAIN_DAT_LEN \ and self.active_index == INDEX_LABEL[0] \ and len(self.training_data) >= TRAIN_DAT_LEN: # swap index self.swap_index(self.DB_disk_location) # init batch write to DB wb_ = self.KV_store.write_batch() for idx_, doc_ in enumerate(documents): cid_ = byt(doc_["CID"]) # resize "code" doc_["code"] = self.resize_vector(doc_["code"], int(os.environ["FIXED_VEC_DIMENSION"])) # cod_ = pickle.dumps(cod_) documents[idx_]["_id"] = next_index # TBD: convert to bulk insert wb_.put(byt(next_index), byt(len(cid_)) + cid_ + CID.doc2bson(doc_)) wb_.put(cid_, byt(next_index)) next_index += 1 wb_.put(byt(-1), byt(next_index)) # commit DB write wb_.write() # push to training data self.update_training_data(documents) # add vectors to index return self.index.add_vectors(documents) def delete_vectors (self, cids): # get ids from cids ids_ = [] wb_ = self.KV_store.write_batch() for cid_ in cids: id_ = self.KV_store.get(byt(cid_)) if id_: ids_.append(int(id_)) wb_.delete(id_) # commit db write wb_.write() # delete vectors by ID from index status, ids = self.index.delete_vectors(ids_) if status and len(ids) == len(cids): return cids else: return [] def get_nearest (self, qmatrix, k, rad): ids = [] dists = [] qmatrix = self.resize_matrix(qmatrix, int(os.environ["FIXED_VEC_DIMENSION"])) # radius defined, if rad is not None: if k is not None: ids, dists = self.index.get_nearest_rad(qmatrix, rad) else: ids, dists = self.index.get_nearest_rad(qmatrix, rad)[:k] else: ids, dists = self.index.get_nearest_k(qmatrix, k) # get docs for idx_, idb in enumerate(ids): for idx__, id_ in enumerate(idb): value = self.KV_store.get(byt(id_)) if value: cid_len_ = int(value[:2]) + 2 ids[idx_][idx__] = CID.bson2doc(value[cid_len_:]) else: ids[idx_][idx__] = None return ids, dists def get_index (self, location): index = None annoy_location = location + "/h_annoy" faiss_location = location + "/h_faiss" if is_mini_instance == "inactive": # try loading faiss index = hfaiss.Faiss(faiss_location) # check if faiss is not loaded, if not index.is_initiated(): # destruct faiss del index # load annoy index = hannoy.Annoy(annoy_location) self.active_index = INDEX_LABEL[0] else: self.active_index = INDEX_LABEL[1] else: # load annoy index = hannoy.Annoy(annoy_location) self.active_index = INDEX_LABEL[0] return index def resize_vector (self, vector, dim): # resize vectors vector_l = len(vector) # check if the vector length is below dimention limit # then pad vector with 0 by dimension if vector_l < dim: vector.extend([0]*(dim-vector_l)) # make sure vector length doesn't exceed dimension limit vector = vector[:dim] return vector def resize_matrix (self, matrix_, dim): # numpize matrix = np.array(matrix_) # check for valid dimensions if matrix.ndim < 2: matrix = np.array([matrix_]) elif matrix.ndim > 2: logging.error("Invalid query dimensions") return [[]] # resize vectors vector_l = len(matrix_[0]) # check if the vector length is below dimention limit # then pad vector with 0 by dimension if vector_l < dim: matrix = np.pad(matrix, ((0, 0), (0, dim-vector_l))) # make sure vector length doesn't exceed dimension limit matrix = matrix[:, :dim] # listize return matrix.tolist() def swap_index (self, location): logging.debug("Swapping index to FAISS") faiss_location = location + "/h_faiss" # init faiss index self.index = hfaiss.Faiss(faiss_location) # train faiss index self.index.init_faiss(self.training_data) # migrate data for idx_ in range(int(self.KV_store.get(byt(-1)))): value = self.KV_store.get(byt(idx_)) if value: cid_len_ = int(value[:2]) + 2 self.index.add_vectors([{ "_id": int(idx_), "code": CID.bson2doc(value[cid_len_:])["code"] }]) # set active index self.active_index = INDEX_LABEL[1] def update_training_data (self, documents): # insert new vectors to array self.pipeline.put(documents) def save_TD_to_disk (self): try: # write index np.save(self.TD_location, np.array(self.training_data)) logging.debug('Training data writing success') return True except Exception as e: logging.error('Training data writing failed' + str(e)) return False def load_TD_from_disk (self): try: # load training data self.training_data = np.load(self.TD_location+'.npy').tolist() logging.debug('Training data loaded successfully') return True except Exception as e: logging.error('Training data loading failed' + str(e)) self.training_data = [] return False def process (self): while (self.process_flag): # set a timeout time.sleep(self.process_timeout_sec) # check if queue is not empty if not self.pipeline.empty(): # fetch all currently available codes from queue while not self.pipeline.empty(): # pop available codes for doc_ in self.pipeline.get_nowait(): self.training_data.append(doc_["code"]) # shuffle array random.shuffle(self.training_data) # resize array self.training_data = self.training_data[:TRAIN_DAT_LEN] # write to disk self.save_TD_to_disk() def spawn (self): # create pipeline to add documents self.pipeline = queue.Queue(maxsize=self.q_maxsize) # create process thread self.process_thread = threading.Thread(target=self.process, args=(), daemon=True) # start process thread self.process_thread.start() ================================================ FILE: aquila/metricstore/router.py ================================================ import logging logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) import manager import os import json from utils import CID, schema STORE_LOCATION = os.environ["DATA_STORE_LOCATION"] # databases dictionary databases = {} def preload_databases (): """ Load available databases from disk """ # list database names for content_ in os.listdir(STORE_LOCATION): # simple validation for database name (CID lenngth) if len(content_) >= 43: # TODO: 43 check is a bad method, replace it soon database_name = content_ with open(STORE_LOCATION + database_name + "/schema.json") as ischema: json_schema = json.load(ischema) manager_h = manager.VecManager(json_schema) # load and create databases dict validator_fn = schema.compile(json_schema) databases[database_name] = { "manager_h": manager_h, "schema": { "json": json_schema, "validator": validator_fn } } logging.debug("Loaded all existing databases") def create_database (json_schema): """ Create a database from a given valid JSON schema """ # TBD: write ahead logging (INIT) # generate proper schema definition from templete schema json_schema = schema.generate_schema(json_schema) # identify invalid schema template if json_schema == None: return None # Check if database already exists database_name = CID.doc2CID(json_schema) if databases.get(database_name): # return database name logging.debug("Database already exists") return database_name # If database doesn't exist already, # then create one manager_h = manager.VecManager(json_schema) database_name = manager_h.database_name validator_fn = schema.compile(json_schema) databases[database_name] = { "manager_h": manager_h, "schema": { "json": json_schema, "validator": validator_fn } } # TBD: save schema to storage # TBD: write ahead logging (END) return database_name def load_database (database_name): """ Load an already existing database """ return databases.get(database_name) def insert_docs (docs, database_name): """ Insert a set of valid documents to database """ # write ahead log (INIT) cids_ = [] docs_ = [] # get manager_h for database_name database_h = load_database(database_name) # invalid database name if not database_h: logging.debug("Database doesn't exist. Please create one.") return cids_ # validate docs against schema # and add CID for doc_ in docs: payload = doc_["payload"] if schema.validate_json_docs(database_h["schema"]["validator"], payload): CID_ = CID.doc2CID(payload) cids_.append(CID_) payload["CID"] = CID_ docs_.append(payload) else: cids_.append(None) # get manager_h for database_name manager_h = database_h["manager_h"] manager_h.add_vectors(docs_) # write ahead log (END) return cids_ def delete_docs (ids, database_name): # write ahead log (INIT) # get manager_h for database_name database_h = load_database(database_name) # invalid database name if not database_h: logging.debug("Database doesn't exist. Please create one.") return [] return database_h["manager_h"].delete_vectors(ids) def search (matrix, k, rad, database_name): # write ahead log (INIT) # get manager_h for database_name database_h = load_database(database_name) return database_h["manager_h"].get_nearest(matrix, k, rad) ================================================ FILE: aquila/metricstore/run_tests.sh ================================================ rm -r /data/* python3 -m unittest test.apis.db_fns -v python3 -m unittest test.apis.doc_fns -v python3 -m unittest test.apis.search_fns -v python3 -m unittest test.apis.auth_fns -v ================================================ FILE: aquila/metricstore/test/__init__.py ================================================ ================================================ FILE: aquila/metricstore/test/apis/auth_fns.py ================================================ import unittest import index from utils import CID, schema from Crypto.Hash import SHA384 from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 import base58 import requests from requests.structures import CaseInsensitiveDict import json import bson from multiprocessing import Process # load private key with open("/ossl/private_unencrypted.pem", "r") as pkf: k = pkf.read() priv_key = RSA.import_key(k) class TestAuth (unittest.TestCase): # A fresh DB is created def test_1_auth_create_db (self): # deploy app server = Process(target=index.flaskserver) server.start() schema_def = { "description": "this is my database", "unique": "r8and0mseEd905", "encoder": "example.com/autoencoder/API", "codelen": 30, "metadata": { "name": "string", "age": "number" } } data_ = { "schema": schema_def } data_bson = bson.dumps(data_) # generate hash hash = SHA384.new() hash.update(data_bson) # Sign with pvt key signer = pkcs1_15.new(priv_key) signature = signer.sign(hash) signature = base58.b58encode(signature).decode("utf-8") url = "http://127.0.0.1:5001/db/create" headers = CaseInsensitiveDict() headers["Content-Type"] = "application/json" data = { "data": data_, "signature": signature } data = json.dumps(data) resp = requests.post(url, headers=headers, data=data) database_name_ = resp.json()["database_name"] schema_def = schema.generate_schema(schema_def) database_name = CID.doc2CID(schema_def) server.terminate() server.join() self.assertEqual(database_name, database_name_, "DB name doesn't match") if __name__ == '__main__': unittest.main() ================================================ FILE: aquila/metricstore/test/apis/db_fns.py ================================================ import unittest import index # includes preloading databases import router from utils import CID, schema class TestDB (unittest.TestCase): # A fresh DB is created def test_1_db_fresh_create (self): schema_def1 = { "description": "this is my database", "unique": "r8and0mseEd90", "encoder": "example.com/autoencoder/API", "codelen": 3, "metadata": { "name": "string", "age": "number" } } schema_def2 = { "description": "this is my database", "unique": "r8and0mseEd90", "encoder": "example.com/autoencoder/API", "codelen": 3, "metadata": { "name": "string", "age": "number" } } database_name = router.create_database(schema_def1) schema_def = schema.generate_schema(schema_def2) database_name_ = CID.doc2CID(schema_def) self.assertEqual(database_name, database_name_, "DB name doesn't match") # An existing DB is created def test_2_db_exist_create (self): schema_def1 = { "description": "this is my database", "unique": "r8and0mseEd90", "encoder": "example.com/autoencoder/API", "codelen": 3, "metadata": { "name": "string", "age": "number" } } schema_def2 = { "description": "this is my database", "unique": "r8and0mseEd90", "encoder": "example.com/autoencoder/API", "codelen": 3, "metadata": { "name": "string", "age": "number" } } database_name = router.create_database(schema_def1) schema_def = schema.generate_schema(schema_def2) database_name_ = CID.doc2CID(schema_def) self.assertEqual(database_name, database_name_, "DB name doesn't match") if __name__ == '__main__': unittest.main() ================================================ FILE: aquila/metricstore/test/apis/doc_fns.py ================================================ import unittest import index # includes preloading databases import router from utils import CID import numpy as np import time class TestDocs (unittest.TestCase): # A fresh doc is created def test_1_doc_fresh_create (self): # create database schema_def = { "description": "this is my database", "unique": "r8and0mseEd901", "encoder": "example.com/autoencoder/API", "codelen": 3, "metadata": { "name": "string", "age": "number" } } database_name = router.create_database(schema_def) # add document docs = [{ "metadata": { "name":"name1", "age": 20 }, "code": [1,2,3] }, { "metadata": { "name":"name2", "age": 30 }, "code": [1,2,3] }] cids = router.insert_docs(docs, database_name) self.assertEqual(len(cids), len(docs), "Document creation failed") # An incomplete doc is created def test_1a_doc_incomplete_create (self): # create database schema_def = { "description": "this is my database", "unique": "r8and0mseEd901", "encoder": "example.com/autoencoder/API", "codelen": 3, "metadata": { "name": "string", "age": "number" } } database_name = router.create_database(schema_def) # add document with "code" missing docs = [{ "metadata": { "name":"name1", "age": 20 } }, { "metadata": { "name":"name2", "age": 30 } }] cids = router.insert_docs(docs, database_name) self.assertEqual(cids, [None, None], "Document creation test failed") # An existing doc is created def test_2_doc_exist_create (self): # create database schema_def = { "description": "this is my database", "unique": "r8and0mseEd901", "encoder": "example.com/autoencoder/API", "codelen": 3, "metadata": { "name": "string", "age": "number" } } database_name = router.create_database(schema_def) # add existing document docs = [{ "metadata": { "name":"name1", "age": 20 }, "code": [1,2,3] }, { "metadata": { "name":"name2", "age": 30 }, "code": [1,2,3] }] cids = router.insert_docs(docs, database_name) self.assertEqual(len(cids), len(docs), "Document creation failed") # A non existing DB is used to create doc def test_3_db_fresh_doc_create (self): # create random DB name database_name = "BRANDOM" # add existing document docs = [{ "metadata": { "name":"name1", "age": 20 }, "code": [1,2,3] }, { "metadata": { "name":"name2", "age": 30 }, "code": [1,2,3] }] cids = router.insert_docs(docs, database_name) self.assertEqual(len(cids), 0, "Document creation failed") # A fresh doc is deleted def test_4_doc_fresh_delete (self): # create database schema_def = { "description": "this is my database", "unique": "r8and0mseEd901fr", "encoder": "example.com/autoencoder/API", "codelen": 3, "metadata": { "name": "string", "age": "number" } } database_name = router.create_database(schema_def) # delete non existing documents cids = router.delete_docs(["sdfsdfsdf", "tret456"], database_name) self.assertEqual(len(cids), 0, "Doc deletion failed") # An existing doc is deleted for small dataset def test_5a_doc_exist_delete_small (self): # create database schema_def = { "description": "this is my database", "unique": "r8and0mseEd902", "encoder": "example.com/autoencoder/API", "codelen": 100, "metadata": { "name": "string", "age": "number" } } database_name = router.create_database(schema_def) # add small epoch document docs = [] # create special doc matrix_r_spec = np.random.rand(1, 100) docs.append({ "metadata": {"name":"special", "age":11}, "code": matrix_r_spec[0].tolist() }) cids_spec = router.insert_docs(docs, database_name) # create other docs for _ in range(90): docs = [] # create random matrix matrix_r = np.random.rand(100, 100) # create documents for item in matrix_r: docs.append({ "metadata": {"name":"generic", "age":10}, "code": item.tolist() }) cids = router.insert_docs(docs, database_name) # check for doc existance k = 2 docs, dists = router.search([matrix_r_spec[0].tolist()], k, None, database_name) self.assertEqual(docs[0][0]["metadata"]["name"], "special", "Doc doesn't exist") # delete special doc cids = router.delete_docs(cids_spec, database_name) time.sleep(10) # check for doc existance k = 2 docs, dists = router.search([matrix_r_spec[0].tolist()], k, None, database_name) self.assertEqual(len(docs[0]), k, "Doc deletion failed") self.assertEqual(docs[0][0]["metadata"]["name"], "generic", "Doc deletion failed") # An existing doc is deleted for large dataset def test_5b_doc_exist_delete_large (self): # create database schema_def = { "description": "this is my database", "unique": "r8and0mseEd902", "encoder": "example.com/autoencoder/API", "codelen": 100, "metadata": { "name": "string", "age": "number" } } database_name = router.create_database(schema_def) # add small epoch document docs = [] # create special doc matrix_r_spec = np.random.rand(1, 100) docs.append({ "metadata": {"name":"special", "age":11}, "code": matrix_r_spec[0].tolist() }) cids_spec = router.insert_docs(docs, database_name) # create other docs for _ in range(120): docs = [] # create random matrix matrix_r = np.random.rand(100, 100) # create documents for item in matrix_r: docs.append({ "metadata": {"name":"generic", "age":10}, "code": item.tolist() }) cids = router.insert_docs(docs, database_name) time.sleep(60) # check for doc existance k = 2 docs, dists = router.search([matrix_r_spec[0].tolist()], k, None, database_name) self.assertEqual(docs[0][0]["metadata"]["name"], "special", "Doc doesn't exist") # delete special doc cids = router.delete_docs(cids_spec, database_name) time.sleep(60) # check for doc existance k = 2 docs, dists = router.search([matrix_r_spec[0].tolist()], k, None, database_name) self.assertEqual(len(docs[0]), k, "Doc deletion failed") self.assertEqual(docs[0][0]["metadata"]["name"], "generic", "Doc deletion failed") # A non existing DB is used to delete doc def test_6_db_fresh_doc_delete (self): # create random DB name database_name = "BRANDOM" # delete non existing documents cids = router.delete_docs(["sdfsdfsdf", "tret456"], database_name) self.assertEqual(len(cids), 0, "Doc deletion failed") if __name__ == '__main__': unittest.main() ================================================ FILE: aquila/metricstore/test/apis/search_fns.py ================================================ import unittest import time import index # includes preloading databases import router from utils import CID import numpy as np class TestSearch (unittest.TestCase): # Perform search on small data def test_1_insert_and_search_small (self): schema_def = { "description": "this is my database", "unique": "r8and0mseEd903", "encoder": "example.com/autoencoder/API", "codelen": 100, "metadata": {} } database_name = router.create_database(schema_def) # insert documents in bulk -------------------- for i_ in range(99): # create random matrix matrix_r = np.random.rand(100, 100) # create documents docs = [] for item in matrix_r: docs.append({ "metadata": {}, "code": item.tolist() }) cids = router.insert_docs(docs, database_name) k = 10 docs, dists = router.search(np.random.rand(1, 784), k, None, database_name) self.assertEqual(k, len(docs[0]), "Search failed") # Perform search on large data def test_2_insert_and_search_large (self): schema_def = { "description": "this is my database", "unique": "r8and0mseEd904", "encoder": "example.com/autoencoder/API", "codelen": 100, "metadata": {} } database_name = router.create_database(schema_def) # insert documents in bulk -------------------- for i_ in range(120): # create random matrix matrix_r = np.random.rand(100, 100) # create documents docs = [] for item in matrix_r: docs.append({ "metadata": {}, "code": item.tolist() }) cids = router.insert_docs(docs, database_name) # time.sleep(5) k = 10 docs, dists = router.search(np.random.rand(1, 784), k, None, database_name) self.assertEqual(k, len(docs[0]), "Search failed") if __name__ == '__main__': unittest.main() ================================================ FILE: aquila/metricstore/utils/CID.py ================================================ import logging import bson import hashlib import base58 def doc2CID (inp): try: # JSON OBJ to BSON encode bson_ = bson.dumps(inp) # SHA-256 Double Hashing hash_ = hashlib.sha256(bson_) hash_ = hashlib.sha256(hash_.digest()) # Convert to Base-58 string b58c_ = base58.b58encode(hash_.digest()) return b58c_.decode('utf-8') except Exception as e: logging.debug(e) return None def doc2bson (inp): try: # JSON OBJ to BSON encode bson_ = bson.dumps(inp) return bson_ except Exception as e: logging.debug(e) return None def bson2doc (inp): try: # BSON to JSON OBJ json_ = bson.loads(inp) return json_ except Exception as e: logging.debug(e) return None ================================================ FILE: aquila/metricstore/utils/config.py ================================================ import yaml import os os.environ["DATA_STORE_LOCATION"] = "/data/" os.environ["LOG_CHUNK_LEN"] = "1000" if "MINI_AQDB" not in os.environ: os.environ["MINI_AQDB"] = "inactive" else: os.environ["MINI_AQDB"] = "active" with open("DB_config.yml", "r") as stream: DB_config = yaml.safe_load(stream) if "AUTH_KEY_FILE" not in os.environ: os.environ["AUTH_KEY_FILE"] = str(DB_config["auth"]["pubkey"]) if "FIXED_VEC_DIMENSION" not in os.environ: os.environ["FIXED_VEC_DIMENSION"] = str(DB_config["docs"]["vd"]) if "MIN_SAWP_COUNT" not in os.environ: os.environ["MIN_SAWP_COUNT"] = str(DB_config["docs"]["cswap"]) if "FIXED_Q_LEN" not in os.environ: os.environ["FIXED_Q_LEN"] = str(DB_config["queue"]["qlen"]) if "THREAD_SLEEP" not in os.environ: os.environ["THREAD_SLEEP"] = str(DB_config["queue"]["sleep"]) if "FAISS_MAX_CELLS" not in os.environ: os.environ["FAISS_MAX_CELLS"] = str(DB_config["faiss"]["init"]["nlist"]) if "FAISS_VISIT_CELLS" not in os.environ: os.environ["FAISS_VISIT_CELLS"] = str(DB_config["faiss"]["init"]["nprobe"]) if "FAISS_BYTES_PER_VEC" not in os.environ: os.environ["FAISS_BYTES_PER_VEC"] = str(DB_config["faiss"]["init"]["bpv"]) if "FAISS_BYTES_PER_SUB_VEC" not in os.environ: os.environ["FAISS_BYTES_PER_SUB_VEC"] = str(DB_config["faiss"]["init"]["bpsv"]) if "ANNOY_SIM_METRIC" not in os.environ: os.environ["ANNOY_SIM_METRIC"] = str(DB_config["annoy"]["init"]["smetric"]) if "ANNOY_NTREES" not in os.environ: os.environ["ANNOY_NTREES"] = str(DB_config["annoy"]["init"]["ntrees"]) if "ANNOY_SK" not in os.environ: os.environ["ANNOY_SK"] = str(DB_config["annoy"]["init"]["search_k"]) ================================================ FILE: aquila/metricstore/utils/cryptops.py ================================================ import logging from Crypto.Hash import SHA384 from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 import base58 import chardet import bson def read_public_key (location): # disk read public key with open(location, "r") as pkf: k = pkf.read() pub_key = RSA.import_key(k) return pub_key def verify_signature (json_data, pub_key, signature): ret = True binary_data = bson.dumps(json_data) # generate hash hash = SHA384.new() hash.update(binary_data) signature = base58.b58decode(signature) # Verify with pub key verifier = pkcs1_15.new(pub_key) try: verifier.verify(hash, signature) except Exception as e: logging.debug(e) ret = False return ret ================================================ FILE: aquila/metricstore/utils/schema.py ================================================ import logging import fastjsonschema def generate_schema (template): # get all keys from template schema metadata_templ = template.get("metadata") if metadata_templ: del template["metadata"] else: metadata_templ = {} keys_ = list(template.keys()) # sort keys keys_.sort() # validate required keys req_k = ["description", "encoder", "unique"] for r_ in req_k: if r_ not in keys_: # cannot continue, invalid schema return None # generate schema in sorted keys generated_schema = { "$schema": "http://json-schema.org/schema#", "$id": "http://aquilanetwork.com/schema/id/v1", "title": "Schema", "description": "", "encoder": "", "unique": "", "type": "object", "properties": { "code": { "description": "Encoded data", "type": "array", "items": { "type": "number" } }, "metadata": { "$ref": "#/definitions/metadata" } }, "definitions": { "metadata": { "description": "User defined metadata", "type": "object", "properties": {}, "required": [] } }, "required": ["code", "metadata"] } metadata = {} metadata_types = ["number", "string"] for key_ in keys_: if key_ in req_k: # fill in root level keys generated_schema[key_] = template[key_] elif key_ == "codelen": generated_schema["properties"]["code"]["maxItems"] = template[key_] else: # fill in root level keys :: extra info that we don't care generated_schema[key_] = template[key_] for key_ in metadata_templ.keys(): # check type is predefined if metadata_templ[key_] not in metadata_types: return None # fill in metadata keys metadata[key_] = { "type": metadata_templ[key_] } generated_schema["definitions"]["metadata"]["properties"] = metadata generated_schema["definitions"]["metadata"]["required"] = list(metadata.keys()) # validate values if type(generated_schema["properties"]["code"].get("maxItems")) != int: return None return generated_schema def compile (schema_def): # compile schema validator = fastjsonschema.compile(schema_def) return validator def validate_json_docs (validator, json_doc): try: # validate doc on schema :: will except on fail validator(json_doc) # validated # logging.debug("Schema validation success") return True except Exception as e: logging.error(e) return False ================================================ FILE: aquila/metricstore/vec_index/hannoy.py ================================================ import logging import numpy as np from annoy import AnnoyIndex import yaml import os import threading import queue import time class Annoy: def __init__(self, model_location): # set model location self.model_location = model_location # to keep the thread & queue running self.process_flag = True self.q_maxsize = int(os.environ["FIXED_Q_LEN"]) self.build_batch_size = int(os.environ["FIXED_Q_LEN"]) self.process_thread = None self._lock = threading.Lock() self.process_timeout_sec = int(os.environ["THREAD_SLEEP"]) # seconds # this is to keep track of all vectors inserted # for saving into disk and retrieve later self.index_disk = None try: # make sure to parse env variables to their expected type self.dim = int(os.environ["FIXED_VEC_DIMENSION"]) self.sim_metric = str(os.environ["ANNOY_SIM_METRIC"]) self.n_trees = int(os.environ["ANNOY_NTREES"]) self.search_k = int(os.environ["ANNOY_SK"]) self.model_loaded = self.load_model_from_disk() if not self.model_loaded: if self.init_annoy(): logging.debug("Annoy Init done") else: logging.debug("Annoy Init Failed") except Exception as e: logging.error('Error initializing Annoy: ' + e) # spawn process thread self.spawn() def __del__(self): self.process_flag = False if self.process_thread: self.process_thread.join() def spawn (self): # create pipeline to add documents self.pipeline = queue.Queue(maxsize=self.q_maxsize) # create process thread self.process_thread = threading.Thread(target=self.process, args=(), daemon=True) # start process thread self.process_thread.start() # return self.pipeline def init_annoy(self): # only do if no index loaded from disk if not self.model_loaded: logging.debug('Annoy init index') self.a_index = AnnoyIndex(self.dim, self.sim_metric) # Lock index read / wtite until it is built with self._lock: # build index build_ = self.a_index.build(self.n_trees) if build_: self.model_loaded = self.save_model_to_disk() return self.model_loaded def add_vectors(self, documents): # add documents to queue self.pipeline.put({"action":"add", "docs": documents}) return True def process(self): while (self.process_flag): # set a timeout till next vector indexing time.sleep(self.process_timeout_sec) # check if queue is not empty if self.pipeline.qsize() > 0: # Lock index read / wtite until it is built with self._lock: # unbuild index first self.a_index.unbuild() len_documents = 0 # fetch all currently available documents from queue while not self.pipeline.empty(): # extract document & contents qitem = self.pipeline.get_nowait() if qitem["action"] == "add": documents = qitem["docs"] len_documents += len(documents) for document in documents: _id = document["_id"] vector_e = document["code"] # add vector to index self.a_index.add_item(int(_id), vector_e) # append to disk proxy if self.index_disk is None: self.index_disk = np.array([vector_e + [int(_id)]], dtype=float) else: self.index_disk = np.append(self.index_disk, [vector_e + [int(_id)]], axis=0) elif qitem["action"] == "delete": ids = qitem["ids"] len_documents += len(ids) # reset zero_ = np.zeros(self.dim + 1) for id_ in ids: # add zero vector to index self.a_index.add_item(int(id_), zero_[:-1].tolist()) # reset npy disk array if self.index_disk is not None: self.index_disk[ids] = zero_ # take a rest if doc length is > batch_size if len_documents > self.build_batch_size: break # build vector build_ = self.a_index.build(self.n_trees, n_jobs=-1) # write to disk if build_: self.model_loaded = self.save_model_to_disk() def delete_vectors(self, ids): # add documents to queue self.pipeline.put({"action":"delete", "ids": ids}) return True, ids def get_nearest_k(self, matrix, k): ids = [] dists = [] # Lock index read / wtite until nearest neighbor search with self._lock: for vec_data in matrix: if self.search_k != -1: _id, _dist = self.a_index.get_nns_by_vector(vec_data, k, self.search_k, include_distances=True) else: _id, _dist = self.a_index.get_nns_by_vector(vec_data, k, include_distances=True) ids.append(_id) dists.append(_dist) return ids, dists def get_nearest_rad(self, matrix, rad): ids = [] dists = [] # Lock index read / wtite until nearest neighbor search with self._lock: for vec_data in matrix: if self.search_k != -1: _id, _dist = self.a_index.get_nns_by_vector(vec_data, k, self.search_k, include_distances=True) else: _id, _dist = self.a_index.get_nns_by_vector(vec_data, k, include_distances=True) ids.append(_id) dists.append(_dist) return ids, dists def load_model_from_disk(self): try: # prepare new index self.a_index = AnnoyIndex(self.dim, self.sim_metric) # read index self.index_disk = np.load(self.model_location+'.npy') # build Annoy Index for vec_ in self.index_disk.tolist(): self.a_index.add_item(int(vec_[-1]), vec_[0:-1]) # build index build_ = self.a_index.build(self.n_trees) logging.debug('Annoy index loading success') return True except Exception as e: logging.debug('Annoy index loading failed. Creating new index..') return False def save_model_to_disk(self): try: # write index np.save(self.model_location, self.index_disk) logging.debug('Annoy index writing success') return True except Exception as e: logging.error('Annoy index writing failed' + e) return False ================================================ FILE: aquila/metricstore/vec_index/hfaiss.py ================================================ import logging import numpy as np import faiss import yaml import os import threading import queue import time class Faiss: def __init__(self, model_location): # set model location self.model_location = model_location self.is_active = False self.nlist = int(os.environ["FAISS_MAX_CELLS"]) self.nprobe = int(os.environ["FAISS_VISIT_CELLS"]) self.bytesPerVec = int(os.environ["FAISS_BYTES_PER_VEC"]) self.bytesPerSubVec = int(os.environ["FAISS_BYTES_PER_SUB_VEC"]) # make sure to parse env variables to their expected type self.dim = int(os.environ["FIXED_VEC_DIMENSION"]) self.model_loaded = self.load_model_from_disk(self.model_location) self.is_initiated_ = self.model_loaded # to keep the thread & queue running self.process_flag = True self.q_maxsize = int(os.environ["FIXED_Q_LEN"]) self.process_thread = None self._lock = threading.Lock() self.process_timeout_sec = int(os.environ["THREAD_SLEEP"]) # seconds # spawn process thread self.spawn() def __del__(self): self.process_flag = False if self.process_thread: self.process_thread.join() def spawn (self): # create pipeline to add documents self.pipeline = queue.Queue(maxsize=self.q_maxsize) # create process thread self.process_thread = threading.Thread(target=self.process, args=(), daemon=True) # start process thread self.process_thread.start() # return self.pipeline def init_faiss(self, matrix): self.train_data = np.matrix(matrix).astype('float32') logging.debug('FAISS init quantizer') self.f_quantizer = faiss.IndexFlatIP(self.dim) # Lock index read / wtite until it is built with self._lock: logging.debug('FAISS init index') self.f_index = faiss.IndexIVFPQ(self.f_quantizer, self.dim, self.nlist, self.bytesPerVec, self.bytesPerSubVec) logging.debug('FAISS train index') self.f_index.train(self.train_data) logging.debug('FAISS train index finished') # write index to disk self.model_loaded = self.save_model_to_disk(self.model_location, self.f_index) self.is_initiated_ = self.model_loaded return self.is_initiated_ def is_initiated(self): return self.is_initiated_ def load_model_from_disk(self, location): try: # read index self.f_index = faiss.read_index(location) logging.debug('FAISS index loading success') return True except Exception as e: logging.error('FAISS index loading failed' + str(e)) return False def save_model_to_disk(self, location, index): try: # write index faiss.write_index(index, location) logging.debug('FAISS index writing success') return True except: logging.error('FAISS index writing failed') return False def add_vectors(self, documents): # add document to queue self.pipeline.put({"action":"add", "docs": documents}) return True def process(self): while (self.process_flag): # set a timeout till next vector indexing time.sleep(self.process_timeout_sec) # check if queue is not empty if self.pipeline.qsize() > 0: ids = [] vecs = [] # f_delete = False # f_add = False # fetch all currently available documents from queue while not self.pipeline.empty(): # extract document & contents qitem = self.pipeline.get_nowait() if qitem["action"] == "add": # f_add = True documents = qitem["docs"] for document in documents: _id = document["_id"] vector_e = document["code"] ids.append(_id) vecs.append(vector_e) elif qitem["action"] == "delete": # f_delete = True ids = qitem["ids"] # remove vectors and add zero reset self.f_index.remove_ids(np.array(ids).astype('int')) vecs = np.zeros((len(ids), self.dim)) # if f_add: # convert to np matrix vec_data = np.matrix(vecs).astype('float32') id_data = np.array(ids).astype('int') # Lock index read / wtite until it is built with self._lock: # add vectors self.f_index.add_with_ids(vec_data, id_data) # if f_delete: # pass # write to disk self.save_model_to_disk(self.model_location, self.f_index) def delete_vectors(self, ids): # remove vectors self.pipeline.put({"action":"delete", "ids": ids}) return True, ids def get_nearest_rad(self, matrix, rad): pass # # convert to np matrix # vec_data = np.matrix(matrix).astype('float32') # # Lock index read / wtite until nearest neighbor search # with self._lock: # D, I = self.f_index.search(vec_data, k) # return I.tolist(), D.tolist() def get_nearest_k(self, matrix, k): # convert to np matrix vec_data = np.matrix(matrix).astype('float32') # Lock index read / wtite until nearest neighbor search with self._lock: D, I = self.f_index.search(vec_data, k) return I.tolist(), D.tolist() ================================================ FILE: aquila/metricstore/wal/walman.py ================================================ import logging import os import time import queue import threading LOG_STORE_LOCATION = os.environ["DATA_STORE_LOCATION"] + "/log/" LOG_CHUNK_LEN = int(os.environ["LOG_CHUNK_LEN"]) PROCESS_TIMEOUT = int(os.environ["THREAD_SLEEP"]) MAX_Q_LEN = int(os.environ["FIXED_Q_LEN"]) process_flag = True pipeline = None process_thread = None # list logs in log directory log_files_asc = os.listdir(LOG_STORE_LOCATION).sort() # load last log chunk last_chunk_name = log_files_asc[-1] last_chunk_data_len = 0 # read last chunk with open(LOG_STORE_LOCATION + last_chunk_name) as flog: # check if the file is full capacity if len(flog.split("\n")) >= LOG_CHUNK_LEN: create_new_chunk() else: last_chunk_data_len = len(flog.split("\n")) def create_new_chunk (): global log_files_asc global last_chunk_name global last_chunk_data_len # name a new log file new_f_name = time.time() log_files_asc.append(new_f_name) last_chunk_name = new_f_name # initialize log file with open(LOG_STORE_LOCATION + last_chunk_name, "a") as aflog: aflog.write("WALOG INIT") last_chunk_data_len = 1 def spawn (): # spawn worker thread global process_flag process_flag = True # create pipeline to add documents global pipeline pipeline = queue.Queue(maxsize=MAX_Q_LEN) # create process thread global process_thread process_thread = threading.Thread(target=process, args=(), daemon=True) # start process thread process_thread.start() def add_log (batch_in): global pipeline pipeline.put(batch_in) return True def terminate (): global process_flag process_flag = False def process (): global pipeline while (process_flag): time.sleep(PROCESS_TIMEOUT) # check if queue is not empty if not pipeline.empty(): # fetch all currently available codes from queue while not pipeline.empty(): # pop available logs for batch in pipeline.get_nowait(): for log_ in batch: # check if the batch length exceeded if last_chunk_data_len + 1 >= LOG_CHUNK_LEN: create_new_chunk() # write log to disk with open(LOG_STORE_LOCATION + last_chunk_name, "a") as aflog: aflog.write("\n" + log_) ================================================ FILE: aquila/metricstore/wal/walmon.py ================================================ ================================================ FILE: aquila/network/go.mod ================================================ module aquilaport go 1.15 require ( github.com/gorilla/mux v1.8.0 github.com/syndtr/goleveldb v1.0.0 go.mongodb.org/mongo-driver v1.4.4 ) ================================================ FILE: aquila/network/go.sum ================================================ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= go.mongodb.org/mongo-driver v1.4.4 h1:bsPHfODES+/yx2PCWzUYMH8xj6PVniPI8DQrsJuSXSs= go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: aquila/network/main.go ================================================ // main.go package main import ( "encoding/json" "fmt" "io/ioutil" "log" "net/http" "net/http/cookiejar" "strings" "time" "github.com/gorilla/mux" "github.com/syndtr/goleveldb/leveldb" "go.mongodb.org/mongo-driver/bson" ) var _localDb, _ = leveldb.OpenFile("./db/_local", nil) var replicationDb, _ = leveldb.OpenFile("./db/replication", nil) var sourceDb, _ = leveldb.OpenFile("./db/source", nil) var jar, err = cookiejar.New(nil) // if err != nil { // fmt.Println(err) // } // Document struct type Document struct { ID string `json:"id"` Title string `json:"title"` Deleted bool `json:"deleted"` Timestamp string `json:"timestamp"` Version string `json:"version"` } func homePage(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Welcome to the HomePage!") fmt.Println("Endpoint Hit: homePage") } func createNewDocument(w http.ResponseWriter, r *http.Request) { // decode json body var documents []Document json.NewDecoder(r.Body).Decode(&documents) // init a batch insert to level batch := new(leveldb.Batch) // insert docs to batch for _, doc := range documents { // update document version doc.Version = string(getVersion(doc)) // convert struct to bson data, err := bson.Marshal(doc) fmt.Println(err) // insert doc batch.Put([]byte(doc.ID), data) } // write batch to level db err := sourceDb.Write(batch, nil) fmt.Println(err) // iterate over leveldb and get key, val iter := sourceDb.NewIterator(nil, nil) for iter.Next() { key := iter.Key() value := iter.Value() var docRet Document // convert bson to byte bson.Unmarshal(value, &docRet) fmt.Println(string(key), docRet) } iter.Release() err = iter.Error() json.NewEncoder(w).Encode(documents) } func deleteDocument(w http.ResponseWriter, r *http.Request) { var ids []string json.NewDecoder(r.Body).Decode(&ids) // init a batch insert to level batch := new(leveldb.Batch) // delete docs batch for _, id := range ids { // delete doc batch.Delete([]byte(id)) } // write batch to level db err := sourceDb.Write(batch, nil) fmt.Println(err) // iterate over leveldb and get key, val iter := sourceDb.NewIterator(nil, nil) for iter.Next() { key := iter.Key() value := iter.Value() var docRet Document // convert bson to byte bson.Unmarshal(value, &docRet) fmt.Println(string(key), docRet) } iter.Release() err = iter.Error() json.NewEncoder(w).Encode(ids) } func getDocuments(selector string) []Document { var documents []Document if selector == "all" { // iterate over leveldb and get key, val iter := sourceDb.NewIterator(nil, nil) for iter.Next() { // key := iter.Key() value := iter.Value() var docRet Document // convert bson to byte bson.Unmarshal(value, &docRet) documents = append(documents, docRet) } iter.Release() err = iter.Error() } return documents } func getVersion(document Document) []byte { // version: timestamp (milliseconds, 13 digits) + deleted var delStatus byte delStatus = 48 if document.Deleted { delStatus = 49 } versionGen := append([]byte(document.Timestamp), delStatus) return versionGen } // =========================== COUCHDB ====================================================================== func authenticate() (int, []byte) { return request("http://127.0.0.1:5984/_session", "POST", "name=admin&password=password", "x-www-form-urlencoded") } func checkDB(dbName string) (int, []byte) { return request("http://127.0.0.1:5984/"+dbName, "HEAD", "", "") } func createDB(dbName string) (int, []byte) { return request("http://127.0.0.1:5984/"+dbName, "PUT", "", "") } func getDBInfo(dbName string) (int, []byte) { return request("http://127.0.0.1:5984/"+dbName, "GET", "", "") } func getReplicationLog(dbName string, logID string) (int, []byte) { return request("http://127.0.0.1:5984/"+dbName+"/_local/"+logID, "GET", "", "") } func addBatchDocs(dbName string, documents []Document) (int, []byte) { data, err := json.Marshal(documents) if err != nil { fmt.Println(err) } dataStr := `{"docs":` + string(data) + `}` return request("http://127.0.0.1:5984/"+dbName+"/_bulk_docs", "POST", dataStr, "application/json") } func getChanges(dbName string) (int, []byte) { return request("http://127.0.0.1:5984/"+dbName+"/_changes?style=all_docs", "GET", "", "") } func ensureCommit(dbName string) (int, []byte) { return request("http://127.0.0.1:5984/"+dbName+"/_ensure_full_commit", "POST", "", "application/json") } func setReplChkPoint(dbName string, replLog []byte) (int, []byte) { return request("http://127.0.0.1:5984/"+dbName+"/_ensure_full_commit", "POST", string(replLog), "application/json") } func request(url string, method string, payload string, contentType string) (int, []byte) { client := &http.Client{ Jar: jar, } var req *http.Request var err error if payload == "" { req, err = http.NewRequest(method, url, nil) } else { req, err = http.NewRequest(method, url, strings.NewReader(payload)) } if err != nil { fmt.Println(err) } if contentType == "x-www-form-urlencoded" { req.Header.Add("Content-Type", "application/x-www-form-urlencoded") } else if contentType == "application/json" { req.Header.Add("Content-Type", "application/json") } res, err := client.Do(req) if err != nil { fmt.Println(err) } defer res.Body.Close() body, err := ioutil.ReadAll(res.Body) if err != nil { fmt.Println(err) } return res.StatusCode, body } func handleRequests() { myRouter := mux.NewRouter().StrictSlash(true) myRouter.HandleFunc("/", homePage) myRouter.HandleFunc("/create", createNewDocument).Methods("POST") myRouter.HandleFunc("/delete", deleteDocument).Methods("POST") log.Fatal(http.ListenAndServe(":10000", myRouter)) } func replicatorDemon() { for { // authenticate couchDB stat, _ := authenticate() if stat != 200 { fmt.Println("CouchDB Authentication failed. Exiting demon.") } else { fmt.Println("CouchDB authentication success.") } // perform replication to target ======================================= // sourceDB := "source" targetDB := "target" // 1. verify peers stat, _ = checkDB(targetDB) if stat == 404 { fmt.Println("Target DB do not exist. Creating it..") // create target database stat, _ = createDB(targetDB) if stat == 201 { fmt.Println("Target DB created.") } else { fmt.Println("Target DB can not be created. Aborting replication..") break } } else if stat == 200 { fmt.Println("Target DB exists") } // 2. get peers information fmt.Println("Getting peers information..") stat, _ = getDBInfo(targetDB) replicationID := "" if stat == 200 { // generate replication ID replicationID = "123456" } // 3. find common ascestry fullReplication := false if replicationID != "" { fmt.Println("Generated replication ID: ", replicationID) // get replication log from target fmt.Println("Getting replication log from target..") stat, rlog := getReplicationLog(targetDB, replicationID) if stat == 200 { // compare logs fmt.Println(string(rlog)) } else if stat == 404 { fullReplication = true fmt.Println("Replication log not available. Full replication needed.") } } else { fmt.Println("Replication ID generation failed. Exiting.. ") break } // 4. locate changed documents var documents []Document if fullReplication { // get all documents documents = getDocuments("all") } else { // Get changed documents in target stat, changes := getChanges(targetDB) if stat == 200 { fmt.Println("Changes: ", changes) } // finalize documents to be replicated documents = getDocuments("all") // TODO: To be changed to selectives // finalize replication if no change found if len(documents) <= 0 { fmt.Println("No more changes to replicate.") } } // 5. replicate changes if len(documents) > 0 { stat, _ := addBatchDocs(targetDB, documents) if stat == 201 { fmt.Println("Documents written succesfully.") } else { fmt.Println("Documents write failed.") } // ensure in commit stat, data := ensureCommit(targetDB) if stat == 201 { fmt.Println("Documents commited succesfully.") } else { fmt.Println("Documents commit failed.", string(data)) break } // set record replication checkpoint stat, _ = setReplChkPoint(targetDB, []byte("")) } // wait 5 seconds before next replication loop time.Sleep(time.Duration(5 * time.Second)) } } func main() { go replicatorDemon() handleRequests() } ================================================ FILE: aquila/search/.dockerignore ================================================ node_modules .vscode .env* dist ================================================ FILE: aquila/search/.nvmrc ================================================ v16.17.0 ================================================ FILE: aquila/search/Dockerfile ================================================ FROM node:16-alpine as builder # Browserless ARG BROWSERLESS_API_KEY=your-api-key-here WORKDIR /app RUN apk add curl bash RUN apk add --no-cache git openssh # install node-prune (https://github.com/tj/node-prune) RUN curl -sfL https://gobinaries.com/tj/node-prune | bash -s -- -b /usr/local/bin COPY . . RUN yarn RUN yarn build RUN npm prune --production # run node prune RUN /usr/local/bin/node-prune FROM node:16-alpine # Application Port ENV PORT=5000 # Database ENV DB_HOST=host.docker.internal ENV DB_PORT=5432 ENV DB_NAME=test_db ENV DB_USERNAME=user ENV DB_PASSWORD=password # Redis ENV REDIS_HOST=redis-19166.c62.us-east-1-4.ec2.cloud.redislabs.com ENV REDIS_PORT=19166 ENV REDIS_USERNAME=default ENV REDIS_PASSWORD=WUFOqcuORH4VoVNKBmzPIQvwsmpHqE5a # Jwt ENV JWT_SECRET=jwtsecret ENV JWT_EXPIRES_IN=2h #Summarizer ENV SUMMARIZER_URL=http://host.docker.internal:5008/process #Aquila ENV AQUILA_DB_HOST=http://host.docker.internal ENV AQUILA_DB_PORT=5001 ENV AQUILA_DB_KEY_PATH=../../private_unencrypted.pem ENV AQUILA_HUB_HOST=http://host.docker.internal ENV AQUILA_HUB_PORT=5002 ENV AQUILA_HUB_KEY_PATH=../../private_unencrypted_hub.pem # Browserless ENV BROWSERLESS_API_KEY=b94a8c0c-3b5c-4342-ab2d-a4b19bf1f13b WORKDIR /app COPY --from=builder /app/dist ./dist COPY --from=builder /app/node_modules ./node_modules EXPOSE 5000 CMD ["node", "./dist/index.js"] ================================================ FILE: aquila/search/README.md ================================================ # AquilaX EE ## ⚠️ Server and other Credentials are in workflow file Never make this repo Public Steps to run this project: 1. Run `yarn` command 2. Setup database settings inside `data-source.ts` file 3. Run `yarn run start:dev` command 4. Generate migration `yarn run typeorm -- migration:generate src/migrations/bookmark --dataSource src/config/db.ts` 5. Run migration `yarn run typeorm -- migration:run --dataSource src/config/db.ts` ## Run using Docker Docker build ```bash docker build -t manekshms/aquila-network-api . docker run -d -p 5000:5000 \ -e DB_HOST=localhost \ -e DB_PORT=5432 \ -e DB_NAME=test_db \ -e DB_USERNAME=test \ -e DB_PASSWORD=example \ -e REDIS_HOST=localhost \ -e REDIS_PORT=6379 \ -e REDIS_USERNAME=default \ -e REDIS_PASSWORD=redis123 \ -e JWT_SECRET=jwtsecret \ -e JWT_EXPIRES_IN=2h \ -e SUMMARIZER_URL=http://localhost:5008/process \ -e AQUILA_DB_HOST=http://localhost.internal \ -e AQUILA_DB_PORT=5001 \ -e AQUILA_DB_KEY_PATH=/ossl/private_unencrypted.pem \ -e AQUILA_HUB_HOST=http://localhost.internal \ -e AQUILA_HUB_PORT=5002 \ -e AQUILA_HUB_KEY_PATH=/ossl/private_unencrypted.pem \ -e ENV BROWSERLESS_API_KEY=key \ -v ${HOME}/aquilax/ossl:/ossl --name manekshms/aquila-network-ui ``` ================================================ FILE: aquila/search/package.json ================================================ { "name": "aquilax-ee", "version": "1.0.0", "description": "AquilaX api", "main": "index.js", "repository": "https://github.com/Aquila-HQ/AquilaX-EE.git", "author": "manekshms ", "scripts": { "build": "tsc", "listen": "node dist/index.js", "start": "yarn listen", "dev": "cross-env NODE_ENV=development nodemon -e './**/*.ts' --exec 'yarn build && yarn listen'", "typeorm": "typeorm-ts-node-commonjs" }, "license": "MIT", "devDependencies": { "@types/body-parser": "^1.19.2", "@types/cors": "^2.8.13", "@types/express": "^4.17.17", "@types/jsonwebtoken": "^9.0.1", "@types/multer": "^1.4.7", "@types/node": "^18.15.0", "@types/uuid": "^9.0.1", "cross-env": "^7.0.3", "nodemon": "^2.0.21", "ts-node": "^10.9.1", "typescript": "^4.9.5" }, "dependencies": { "@bull-board/express": "^5.0.0", "@postlight/parser": "^2.2.3", "aquila-js": "^1.0.3", "axios": "^1.3.4", "body-parser": "^1.20.2", "bs58": "^5.0.0", "bullmq": "^3.10.1", "cheerio": "^1.0.0-rc.12", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cors": "^2.8.5", "express": "^4.18.2", "express-validator": "^6.15.0", "ioredis": "^5.3.1", "jsonwebtoken": "^9.0.0", "multer": "^1.4.5-lts.1", "pg": "^8.10.0", "puppeteer-core": "^19.7.4", "reflect-metadata": "^0.1.13", "routing-controllers": "^0.10.2", "typedi": "^0.10.0", "typeorm": "^0.3.12", "uuid": "^9.0.0" } } ================================================ FILE: aquila/search/src/@types/express/index.d.ts ================================================ import { AccountStatus } from "../../service/dto/AuthServiceDto"; declare global { namespace Express { interface Request { token?: string; jwtTokenPayload?: { customerId: string; accountStatus: AccountStatus } } } } ================================================ FILE: aquila/search/src/@types/index.d.ts ================================================ declare module '@postlight/parser' { export type ExtractorArgs = any; export type ParserOptions = { html?: string; fetchAllPages?: boolean; fallback?: boolean; contentType?: 'html' | 'markdown' | 'text'; headers?: Record; extend?: boolean; customExtractor?: (args: ExtractorArgs) => ParserResult; }; export type ParserResult = { title: string; content: string; author: string; date_published: string; lead_image_url: string; dek: string; next_page_url: string; url: string; domain: string; excerpt: string; word_count: number; direction: string; total_pages: number; rendered_pages: number; }; export function parse( url: string, opts: ParserOptions ): Promise; const exported: { parse: typeof parse; }; export default exported; } ================================================ FILE: aquila/search/src/app.ts ================================================ import 'reflect-metadata'; import express from 'express'; import { Container } from 'typedi'; import { useExpressServer, useContainer} from 'routing-controllers'; import { BullAdapter } from '@bull-board/api/bullAdapter'; import { ExpressAdapter } from '@bull-board/express'; import { createBullBoard } from '@bull-board/api'; import db from './config/db'; import redisConnection from './config/redisConnection'; import { AquilaClientService } from './lib/AquilaClientService'; import { authorizationChecker } from './helper/auth'; import { currentUserChecker } from './helper/user'; import { getAppWorker } from './job/appWorker'; import { AppQueue } from './job/AppQueue'; export default async function main() { useContainer(Container); const app = express(); app.use(express.json()); useExpressServer(app, { cors: true, authorizationChecker, currentUserChecker, middlewares: [`${__dirname}/middleware/**/*.{ts,js}`], controllers: [`${__dirname}/controller/*.js`], defaultErrorHandler: false }); await db.initialize(); const appWorker = getAppWorker(redisConnection); const aqc = Container.get(AquilaClientService); await aqc.connect(); const serverAdapter = new ExpressAdapter(); serverAdapter.setBasePath('/admin/queues'); const appQueue = Container.get(AppQueue); const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({ queues: [new BullAdapter(appQueue.queue)], serverAdapter: serverAdapter, }); app.use('/admin/queues', serverAdapter.getRouter()); return app; } ================================================ FILE: aquila/search/src/config/db.ts ================================================ import { DataSource, DataSourceOptions } from "typeorm"; import { ConfigService } from "../lib/ConfigService"; const configService = new ConfigService(); const dataSourceOptions: DataSourceOptions = { type: 'postgres', host: configService.get('DB_HOST'), port: parseInt(configService.get('DB_PORT'), 10), username: configService.get('DB_USERNAME'), password: configService.get('DB_PASSWORD'), database: configService.get('DB_NAME'), synchronize: false, entities: [`${__dirname}/../entity/**/*.{ts,js}`], migrations: [`${__dirname}/../migrations/**/*.{ts,js}`], migrationsTableName: "migrations", } const dataSource = new DataSource(dataSourceOptions); export default dataSource; ================================================ FILE: aquila/search/src/config/redisConnection.ts ================================================ import Redis from 'ioredis'; import { ConfigService } from '../lib/ConfigService'; const configService = new ConfigService(); const redisConnection = new Redis({ host: configService.get('REDIS_HOST'), port: parseInt(configService.get('REDIS_PORT'), 10), username: configService.get('REDIS_USERNAME'), password: configService.get('REDIS_PASSWORD') }) export default redisConnection; ================================================ FILE: aquila/search/src/controller/AuthController.ts ================================================ import { Body, JsonController, Post } from "routing-controllers"; import { Service } from "typedi"; import { AuthService } from "../service/AuthService"; import { LoginCustomerRequestBodyDto, LoginCustomerResponseBodyDto } from "./dto/AuthControllerDto"; @Service() @JsonController('/auth') export class AuthController { public constructor(private authService: AuthService) {} @Post('/login') public async login( @Body() body: LoginCustomerRequestBodyDto ): Promise { const token = await this.authService.login(body.secretKey); return { token } } } ================================================ FILE: aquila/search/src/controller/BookmarkController.ts ================================================ import { Body, Get, JsonController, Param, Post, QueryParams, UseBefore } from "routing-controllers"; import { Service } from "typedi"; import { Bookmark } from "../entity/Bookmark"; import { BookmarkTemp } from "../entity/BookmarkTemp"; import { JwtPayloadData } from "../helper/decorators/jwtPayloadData"; import { AuthMiddleware } from "../middleware/global/AuthMiddleware"; import { AddBookmarkValidator } from "../middleware/validator/bookmark/AddBookmarkValidator"; import { GetBookmarkByCollectionIdValidator } from "../middleware/validator/bookmark/GetBookmarkByCollectionIdValidator"; import { GetFeaturedBookmarkValidator } from "../middleware/validator/bookmark/GetFeaturedBookmarkValidator"; import { GetPublicBookmarkByCollectionIdParamValidator } from "../middleware/validator/bookmark/GetPublicBookmarkByCollectionIdParamValidator"; import { GetPublicBookmarkByCollectionIdValidator } from "../middleware/validator/bookmark/GetPublicBookmarkByCollectionIdValidator"; import { BookmarkService } from "../service/BookmarkService"; import { AccountStatus, JwtPayload } from "../service/dto/AuthServiceDto"; import { GetBookmarksByCollectionIdOptionsInputDto, GetFeaturedBookmarksOptionsInputDto } from "../service/dto/BookmarkServiceDto"; import { AddBookmarkReqBodyDto, GetBookmarksByCollectionIdReqQueryParamsDto, GetBookmarksByCollectionIdResBodyDto, GetFeaturedBookmarksReqQueryParamsDto, GetFeaturedBookmarksResBodyDto } from "./dto/BookmarkControllerDto"; @Service() @JsonController('/bookmark') export class BookmarkController { public constructor(private bookmarkService: BookmarkService) {} @UseBefore(AuthMiddleware, AddBookmarkValidator) @Post('/') public async addBookmark( @Body() body: AddBookmarkReqBodyDto, @JwtPayloadData() JwtPayloadData: JwtPayload ): Promise { return await this.bookmarkService.addBookmark(body, JwtPayloadData.accountStatus); } @UseBefore(AuthMiddleware, GetBookmarkByCollectionIdValidator) @Get('/:collectionId/search') public async getBookmarksByCollectionId( @Param('collectionId') collectionId: string, @QueryParams() queryParams: GetBookmarksByCollectionIdReqQueryParamsDto, @JwtPayloadData() JwtPayloadData: JwtPayload ): Promise { const options: GetBookmarksByCollectionIdOptionsInputDto = { limit: queryParams.limit ? parseInt(queryParams.limit, 10) : 10, page: queryParams.page ? parseInt(queryParams.page, 10) : 1 } if(queryParams.query) { options.query = queryParams.query; } return this.bookmarkService.getBookmarksByCollectionId(collectionId, options, JwtPayloadData.accountStatus); } @UseBefore(GetPublicBookmarkByCollectionIdValidator, GetPublicBookmarkByCollectionIdParamValidator) @Get('/public/:collectionId/search') public async getPublicBookmarksByCollectionId( @Param('collectionId') collectionId: string, @QueryParams() queryParams: GetBookmarksByCollectionIdReqQueryParamsDto, ): Promise { const options: GetBookmarksByCollectionIdOptionsInputDto = { limit: queryParams.limit ? parseInt(queryParams.limit, 10) : 10, page: queryParams.page ? parseInt(queryParams.page, 10) : 1 } if(queryParams.query) { options.query = queryParams.query; } return this.bookmarkService.getBookmarksByCollectionId(collectionId, options, AccountStatus.PERMANENT); } @UseBefore(GetFeaturedBookmarkValidator) @Get('/public/featured') public async getAllFeaturedBookmark( @QueryParams() queryParams: GetFeaturedBookmarksReqQueryParamsDto ): Promise { const options: GetFeaturedBookmarksOptionsInputDto = { limit: queryParams.limit ? parseInt(queryParams.limit, 10) : 10, page: queryParams.page ? parseInt(queryParams.page, 10) : 1 } return this.bookmarkService.getFeaturedBookmarks(options); } } ================================================ FILE: aquila/search/src/controller/CollectionController.ts ================================================ import { Authorized, Get, JsonController, Param, QueryParams, UseBefore } from "routing-controllers"; import { Service } from "typedi"; import { Collection } from "../entity/Collection"; import { CollectionTemp } from "../entity/CollectionTemp"; import { JwtPayloadData } from "../helper/decorators/jwtPayloadData"; import { GetAllFeaturedCollectionValidator } from "../middleware/validator/collection/GetAllFeaturedCollectionValidator"; import { GetAllPublicCollectionValidator } from "../middleware/validator/collection/GetAllPublicCollectionValidator"; import { GetPublicCollectionByIdValidator } from "../middleware/validator/collection/GetPublicCollectionByIdValidator"; import { CollectionService } from "../service/CollectionService"; import { AccountStatus, JwtPayload } from "../service/dto/AuthServiceDto"; import { GetAllFeaturedCollectionReqQueryParamsDto, GetAllFeaturedCollectionResPayloadDto, GetAllPublicCollectionReqQueryParamsDto, GetAllPublicCollectionResPayloadDto } from "./dto/CollectionControllerDto"; @Service() @JsonController('/collection') export class CollectionController { public constructor(private collectionService: CollectionService) {} @UseBefore(GetAllPublicCollectionValidator) @Get('/public') public async getAllPublicCollection( @QueryParams() queryParams: GetAllPublicCollectionReqQueryParamsDto ): Promise { const options = { limit: queryParams.limit? parseInt(queryParams.limit) : 10, page: queryParams.page ? parseInt(queryParams.page) : 1, where: { isShareable: true } }; return await this.collectionService.getAllCollections(options, AccountStatus.PERMANENT); } @UseBefore(GetAllFeaturedCollectionValidator) @Get('/public/featured') public async getAllPublicFeaturedCollection( @QueryParams() queryParams: GetAllFeaturedCollectionReqQueryParamsDto ): Promise { const options = { limit: queryParams.limit? parseInt(queryParams.limit) : 10, page: queryParams.page ? parseInt(queryParams.page) : 1, where: { isShareable: true, isFeatured: true } }; return await this.collectionService.getAllCollections(options, AccountStatus.PERMANENT); } @UseBefore(GetPublicCollectionByIdValidator) @Get('/public/:collectionId') public async getPublicCollectionById( @Param('collectionId') collectionId: string ): Promise { return await this.collectionService.getPublicCollectionById(collectionId); } @Authorized() @Get('/my-collections') public async getCurrentCustomerCollections( @JwtPayloadData() jwtPayload: JwtPayload ): Promise { const collections = await this.collectionService.getCustomerCollectionsByCustomerId(jwtPayload.customerId, jwtPayload.accountStatus); return collections; } } ================================================ FILE: aquila/search/src/controller/CollectionSubscriptionController.ts ================================================ import { Body, Get, JsonController, Param, Post, QueryParams, UseBefore } from "routing-controllers"; import { Service } from "typedi"; import { Collection } from "../entity/Collection"; import { CollectionSubscription } from "../entity/CollectionSubscription"; import { CollectionSubscriptionTemp } from "../entity/CollectionSubscriptionTemp"; import { JwtPayloadData } from "../helper/decorators/jwtPayloadData"; import { AuthMiddleware } from "../middleware/global/AuthMiddleware"; import { SubscribeCollectionValidator } from "../middleware/validator/csubscription/SubscribeCollectionValidator"; import { UnSubscribeCollectionValidator } from "../middleware/validator/csubscription/UnSubscribeCollectionValidator"; import { CollectionSubscriptionService } from "../service/CollectionSubscriptionService"; import { JwtPayload } from "../service/dto/AuthServiceDto"; import { GetSubscriptionsByCustomerIdOptionsInputDto } from "../service/dto/CollectionSubscriptionServiceDto"; import { GetSubscriptionsByCustomerIdReqQueryParamsDto, GetSubscriptionsByCustomerIdResBodyDto } from "./dto/CollectionSubscriptionControllerDto"; @Service() @JsonController('/subscription') export class CollectionSubscriptionController { public constructor(private collectionSubscriptionService: CollectionSubscriptionService) {} @UseBefore(AuthMiddleware) @Get('/list') public async getCurrentCustomerSubscriptions( @JwtPayloadData() JwtPayloadData: JwtPayload ): Promise { return await this.collectionSubscriptionService.getCustomerSubscriptions(JwtPayloadData.customerId, JwtPayloadData.accountStatus); } @UseBefore(AuthMiddleware, SubscribeCollectionValidator) @Post('/:collectionId/add') public async subscribeCollection( @Param('collectionId') collectionId: string, @JwtPayloadData() JwtPayloadData: JwtPayload ): Promise { return await this.collectionSubscriptionService.subscribeCollection(collectionId, JwtPayloadData.customerId, JwtPayloadData.accountStatus); } @UseBefore(AuthMiddleware) @Post('/:collectionId/is-subscribed') public async isCollectionAlreadySubscribed( @Param('collectionId') collectionId: string, @JwtPayloadData() jwtPayloadData: JwtPayload ):Promise<{ isSubscribed: boolean}> { const isSubscribed = await this.collectionSubscriptionService.isCollectionSubscribedByCustomer(collectionId, jwtPayloadData.customerId, jwtPayloadData.accountStatus); return { isSubscribed }; } @UseBefore(AuthMiddleware, UnSubscribeCollectionValidator) @Post('/:collectionId/remove') public async unSubscribeCollection( @Param('collectionId') collectionId: string, @JwtPayloadData() JwtPayloadData: JwtPayload ): Promise { return await this.collectionSubscriptionService.unSubscribeCollection(collectionId, JwtPayloadData.customerId, JwtPayloadData.accountStatus); } @Get('/') @UseBefore(AuthMiddleware) public async getSubscriptionsByCustomerId( @JwtPayloadData() jwtPayloadData: JwtPayload, @QueryParams() queryParams: GetSubscriptionsByCustomerIdReqQueryParamsDto, ): Promise { const options: GetSubscriptionsByCustomerIdOptionsInputDto = { limit: queryParams.limit ? parseInt(queryParams.limit, 10) : 10, page: queryParams.page ? parseInt(queryParams.page, 10) : 1 } if(queryParams.query) { options.query = queryParams.query; } return await this.collectionSubscriptionService.getSubscriptionsByCustomerId(jwtPayloadData.customerId, options, jwtPayloadData.accountStatus); } @UseBefore(AuthMiddleware) @Get('/collections') public async getCustomerSubscribedCollections( @JwtPayloadData() jwtPayloadData: JwtPayload, ): Promise { return await this.collectionSubscriptionService.getCustomerSubscribedCollections(jwtPayloadData.customerId, jwtPayloadData.accountStatus); } } ================================================ FILE: aquila/search/src/controller/CustomerController.ts ================================================ import { Authorized, Body, CurrentUser, Get, JsonController, Param, Patch, Post, Put, UseBefore } from "routing-controllers"; import { Service } from "typedi"; import { Customer } from "../entity/Customer"; import { CustomerTemp } from "../entity/CustomerTemp"; import { JwtPayloadData } from "../helper/decorators/jwtPayloadData"; import { AuthMiddleware } from "../middleware/global/AuthMiddleware"; import { ActivateCustomerValidator } from "../middleware/validator/customer/ActivateCustomerValidator"; import { CreateCustomerValidator } from "../middleware/validator/customer/CreateCustomerValidator"; import { GetCustomerPublicInfoByIdValidator } from "../middleware/validator/customer/GetCustomerPublicInfoByIdValidator"; import { UpdateCustomerValidator } from "../middleware/validator/customer/UpdateCustomerValidator"; import { CustomerService } from "../service/CustomerService"; import { JwtPayload } from "../service/dto/AuthServiceDto"; import { ActivateCustomerReqBodyDto, CreateCustomerReqBodyDto, CreateCustomerResponseDto, GetCustomerPublicInfoByIdRespBodyDto, GetRandomCustomerNameRespBodyDto, UpdateCustomerReqBodyDto } from "./dto/CustomerControllerDto"; @Service() @JsonController('/customer') export class CustomerControler { public constructor(private customerService: CustomerService){} @UseBefore(AuthMiddleware, UpdateCustomerValidator) @Patch('/') public async updateCustomer( @JwtPayloadData() JwtPayloadData: JwtPayload, @Body() body: UpdateCustomerReqBodyDto ):Promise { return await this.customerService.updateCustomerById(JwtPayloadData.customerId, body); } @UseBefore(GetCustomerPublicInfoByIdValidator) @Get('/public/:customerId') public async getCustomerPublicInfoById( @Param('customerId') customerId: string ): Promise { return await this.customerService.getCustomerPublicInfoById(customerId); } @Authorized() @Get('/me') public async getCustomerById( @CurrentUser() customer: Customer | CustomerTemp ): Promise { return customer; } @UseBefore(CreateCustomerValidator) @Post('/') public async createCustomer( @Body() body: CreateCustomerReqBodyDto ): Promise { return await this.customerService.createCustomer(body); } @UseBefore(AuthMiddleware,ActivateCustomerValidator) @Post('/activate') public async activateCustomer( @JwtPayloadData() jwtPayloadData: JwtPayload, @Body() body: ActivateCustomerReqBodyDto ): Promise { return await this.customerService.activateCustomerById(jwtPayloadData.customerId, body); } @Get('/generate-name') public async generateRandomCustomerName( ): Promise { return await this.customerService.getRandomCustomerName(); } } ================================================ FILE: aquila/search/src/controller/HomeController.ts ================================================ import { Get, JsonController } from "routing-controllers"; import { Service } from "typedi"; import { Queue } from 'bullmq'; @Service() @JsonController('/') export class HomeController { @Get('/') public index() { const queue = new Queue('Paint', { connection: { host: "redis-17256.c17.us-east-1-4.ec2.cloud.redislabs.com", port: 17256, username: 'default', password: 'AbcRVEZFVHYjRNPFxAwLcSdUCA4DuUll' }}); queue.add('cars', { color: 'blue' }); return { msg: "Welcome to Aquila X" } } } ================================================ FILE: aquila/search/src/controller/dto/AuthControllerDto.ts ================================================ import { AccountStatus } from "../../service/dto/AuthServiceDto"; export interface LoginCustomerRequestBodyDto { secretKey: string; } export interface LoginCustomerResponseBodyDto { token: string; } ================================================ FILE: aquila/search/src/controller/dto/BookmarkControllerDto.ts ================================================ export interface AddBookmarkReqBodyDto { html?: string; url: string; collectionId: string; } export interface GetBookmarksByCollectionIdReqQueryParamsDto { query?: string; limit?: string; page?: string; } export interface GetBookmarksByCollectionIdResBodyDto { totalPages: number; totalRecords: number; currentPage: number; limit: number; bookmarks: { id: string; collectionId: string; url: string; title: string; author: string; coverImg: string; summary: string description: string; }[] } export interface GetFeaturedBookmarksReqQueryParamsDto { limit?: string; page?: string; } export interface GetFeaturedBookmarksResBodyDto { totalPages: number; totalRecords: number; currentPage: number; limit: number; bookmarks: { id: string; collectionId: string; url: string; title: string; author: string; coverImg: string; summary: string description: string; }[] } ================================================ FILE: aquila/search/src/controller/dto/CollectionControllerDto.ts ================================================ import { Collection } from "../../entity/Collection"; import { CollectionTemp } from "../../entity/CollectionTemp"; export interface GetAllPublicCollectionReqQueryParamsDto { limit?: string; page?: string; } export interface GetAllPublicCollectionResPayloadDto { totalRecords: number; totalPages: number; currentPage: number, limit: number; collections: Collection[] | CollectionTemp[] } export interface GetAllFeaturedCollectionReqQueryParamsDto { limit?: string; page?: string; } export interface GetAllFeaturedCollectionResPayloadDto { totalRecords: number; totalPages: number; currentPage: number, limit: number; collections: Collection[] | CollectionTemp[] } ================================================ FILE: aquila/search/src/controller/dto/CollectionSubscriptionControllerDto.ts ================================================ export interface GetSubscriptionsByCustomerIdReqQueryParamsDto { query?: string; limit?: string; page?: string; } export interface GetSubscriptionsByCustomerIdResBodyDto { totalPages: number; totalRecords: number; currentPage: number; limit: number; bookmarks: { id: string; collectionId: string; url: string; title: string; author: string; coverImg: string; summary: string description: string; }[] } ================================================ FILE: aquila/search/src/controller/dto/CustomerControllerDto.ts ================================================ import { CollectionTemp } from "../../entity/CollectionTemp"; import { CustomerTemp } from "../../entity/CustomerTemp"; export interface CreateCustomerReqBodyDto { firstName: string; lastName: string; } export interface CreateCustomerResponseDto { customer: CustomerTemp, collection: CollectionTemp } export interface ActivateCustomerReqBodyDto { firstName: string; lastName: string; email: string; desc: string; lightningAddress: string; } export interface UpdateCustomerReqBodyDto { firstName: string; lastName: string; email: string; desc: string; lightningAddress: string; } export interface GetCustomerPublicInfoByIdRespBodyDto { id: string; firstName: string; lastName: string; desc: string; customerId: number; } export interface GetRandomCustomerNameRespBodyDto { firstName: string; lastName: string; } ================================================ FILE: aquila/search/src/entity/Bookmark.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; export enum BookmarkStatus { NOT_PROCESSED = 'NOT_PROCESSED', PROCESSING = 'PROCESSING', PROCESSED = 'PROCESSED', } @Entity({ name: 'bookmark'}) export class Bookmark extends BaseEntity { @PrimaryGeneratedColumn('uuid') public id: string; @Column({ name: 'collection_id', type: 'uuid', nullable: false}) public collectionId: string; @Column({ type: 'varchar', length: 2048, nullable: false}) public url: string; @Column({ type: 'text', nullable: false}) public html: string; @Column({ type: 'varchar', length: 2048, nullable: true}) public title: string; @Column({ type: 'varchar', length: 100, nullable: true}) public author: string; @Column({ name: 'cover_img', type: 'varchar', length: 2048, nullable: true}) public coverImg: string; @Column({ type: 'text', nullable: true}) public summary: string; @Column({ type: 'varchar', length: 2048, nullable: true}) public links: string; @Column({ name: 'is_hidden', type: 'boolean', default: false}) public isHidden: boolean; @Column({ type: 'enum', enum: BookmarkStatus, default: BookmarkStatus.NOT_PROCESSED}) public status: BookmarkStatus; @CreateDateColumn({ name: 'created_at'}) public createdAt: Date; @UpdateDateColumn({name: 'updated_at'}) public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/BookmarkPara.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; @Entity('bookmark_para') export class BookmarkPara extends BaseEntity { @PrimaryGeneratedColumn('uuid') public id: string; @Column({ name: 'bookmark_id', nullable: false}) public bookmarkId: string; @Column({ type: 'text', nullable: false}) public content: string; @CreateDateColumn() public createdAt: Date; @UpdateDateColumn() public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/BookmarkParaTemp.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; @Entity('bookmark_para_temp') export class BookmarkParaTemp extends BaseEntity { @PrimaryGeneratedColumn('uuid') public id: string; @Column({ name: 'bookmark_id', nullable: false}) public bookmarkId: string; @Column({ type: 'text', nullable: false}) public content: string; @CreateDateColumn() public createdAt: Date; @UpdateDateColumn() public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/BookmarkTemp.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; export enum BookmarkTempStatus { NOT_PROCESSED = 'NOT_PROCESSED', PROCESSING = 'PROCESSING', PROCESSED = 'PROCESSED', } @Entity({ name: 'bookmark_temp'}) export class BookmarkTemp extends BaseEntity { @PrimaryGeneratedColumn('uuid') public id: string; @Column({ name: 'collection_id', type: 'uuid', nullable: false}) public collectionId: string; @Column({ type: 'varchar', length: 2048, nullable: false}) public url: string; @Column({ type: 'text', nullable: false}) public html: string; @Column({ type: 'varchar', length: 100, nullable: true}) public title: string; @Column({ type: 'varchar', length: 100, nullable: true}) public author: string; @Column({ name: 'cover_img', type: 'varchar', length: 255, nullable: true}) public coverImg: string; @Column({ type: 'varchar', length: 255, nullable: true}) public summary: string; @Column({ type: 'varchar', length: 255, nullable: true}) public links: string; @Column({ name: 'is_hidden', type: 'boolean', default: false}) public isHidden: boolean; @Column({ type: 'enum', enum: BookmarkTempStatus, default: BookmarkTempStatus.NOT_PROCESSED}) public status: BookmarkTempStatus; @CreateDateColumn({ name: 'created_at'}) public createdAt: Date; @UpdateDateColumn({name: 'updated_at'}) public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/Collection.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, JoinColumn, JoinTable, ManyToOne, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; import { Customer } from "./Customer"; @Entity({ name: 'collection'}) export class Collection extends BaseEntity { @PrimaryGeneratedColumn("uuid") public id: string; @Column({ type: 'varchar', length: 25}) public name: string; @Column({ type: 'varchar', length: 255}) public desc: string; @Column({ name: 'customer_id', type: 'uuid' }) public customerId: string; @ManyToOne(type => Customer) @JoinColumn({ name: 'customer_id', referencedColumnName: 'id'}) public customer: Customer; @Column({ name: 'aquila_db_name', type: 'varchar', length: 255, unique: true}) public aquilaDbName: string; @Column({ name: 'is_shareable', type: 'boolean', default: true}) public isShareable: boolean; @Column({ name: 'indexed_docs_count', type: 'bigint', default: 0}) public indexedDocsCount: number; @Column({ name: 'is_featured', type: 'boolean', default: false}) public isFeatured: boolean; @CreateDateColumn({ name: 'created_at'}) public createdAt: Date; @UpdateDateColumn({ name: 'updated_at'}) public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/CollectionSubscription.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; @Entity({ name: 'collection_subscription'}) export class CollectionSubscription extends BaseEntity { @PrimaryGeneratedColumn('uuid') public id: string; @Column({ name: 'collection_id', type: 'uuid', nullable: false}) public collectionId: string; @Column({ name: 'subscriber_id', type: 'uuid', nullable: false}) public subscriberId: string; @CreateDateColumn({ name: 'subscribed_at'}) public subscribedAt: Date; @CreateDateColumn({ name: 'created_at'}) public createdAt: Date; @UpdateDateColumn({name: 'updated_at'}) public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/CollectionSubscriptionTemp.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; @Entity({ name: 'collection_subscription_temp'}) export class CollectionSubscriptionTemp extends BaseEntity { @PrimaryGeneratedColumn('uuid') public id: string; @Column({ name: 'collection_id', type: 'uuid', nullable: false}) public collectionId: string; @Column({ name: 'subscriber_id', type: 'uuid', nullable: false}) public subscriberId: string; @CreateDateColumn({ name: 'subscribed_at'}) public subscribedAt: Date; @CreateDateColumn({ name: 'created_at'}) public createdAt: Date; @UpdateDateColumn({name: 'updated_at'}) public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/CollectionTemp.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; @Entity({ name: 'collection_temp'}) export class CollectionTemp extends BaseEntity { @PrimaryGeneratedColumn("uuid") public id: string; @Column({ type: 'varchar', length: 25}) public name: string; @Column({ type: 'varchar', length: 255}) public desc: string; @Column({ name: 'customer_id', type: 'uuid' }) public customerId: string; @Column({ name: 'aquila_db_name', type: 'varchar', length: 255, unique: true}) public aquilaDbName: string; @Column({ name: 'is_shareable', type: 'boolean', default: true}) public isShareable: boolean; @Column({ name: 'indexed_docs_count', type: 'bigint', default: 0}) public indexedDocsCount: number; @CreateDateColumn({ name: 'created_at'}) public createdAt: Date; @UpdateDateColumn({ name: 'updated_at'}) public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/Customer.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, Generated, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; @Entity({ name: 'customer'}) export class Customer extends BaseEntity { @PrimaryGeneratedColumn("uuid") public id: string; @Generated("increment") @Column({ name: "customer_id", type: 'bigint'}) public customerId: number; @Column({ name: 'first_name', type: 'varchar', length: 20 }) public firstName: string; @Column({ name: 'last_name', type: 'varchar', length: 20 }) public lastName: string; @Column({ type: 'varchar', length: 255}) public avatar: string; @Column({ type: 'varchar', length: 50}) public email: string; @Column({ type: 'varchar', length: 255}) public desc: string; @Column({ name: 'secret_key', type: 'varchar', length: 255, unique: true }) public secretKey: string; @Column({ name: 'lightning_address', type: 'varchar', length: 255, unique: true, default: null }) public lightningAddress: string | null; @Column({ name: 'is_active', type: 'boolean', default: true }) public isActive: boolean; @CreateDateColumn({ name: 'created_at'}) public createdAt: Date; @UpdateDateColumn({ name: 'updated_at'}) public updatedAt: Date; } ================================================ FILE: aquila/search/src/entity/CustomerTemp.ts ================================================ import { BaseEntity, Column, CreateDateColumn, Entity, Generated, PrimaryGeneratedColumn, UpdateDateColumn } from "typeorm"; @Entity({ name: 'customer_temp'}) export class CustomerTemp extends BaseEntity { @PrimaryGeneratedColumn("uuid") public id: string; @Generated("increment") @Column({ name: "customer_id", type: 'bigint'}) public customerId: number; @Column({ name: 'first_name', type: 'varchar', length: 20 }) public firstName: string; @Column({ name: 'last_name', type: 'varchar', length: 20 }) public lastName: string; @Column({ type: 'varchar', length: 255}) public avatar: string; @Column({ type: 'varchar', length: 255}) public desc: string; @Column({ name: 'secret_key', type: 'varchar', length: 255, unique: true }) public secretKey: string; @Column({ name: 'is_active', type: 'boolean', default: true }) public isActive: boolean; @CreateDateColumn({ name: 'created_at'}) public createdAt: Date; @UpdateDateColumn({ name: 'updated_at'}) public updatedAt: Date; } ================================================ FILE: aquila/search/src/helper/auth.ts ================================================ import { Action } from "routing-controllers"; import Container from "typedi"; import { AuthService } from "../service/AuthService"; export function authorizationChecker(action: Action): boolean { const token = (action.request.headers.authorization || "").replace('Bearer ', '') || ''; const authService = Container.get(AuthService); authService.authenticate(token); return true; } ================================================ FILE: aquila/search/src/helper/decorators/jwtPayloadData.ts ================================================ import { Action, createParamDecorator } from "routing-controllers"; import Container from "typedi"; import { AuthService } from "../../service/AuthService"; export function JwtPayloadData() { return createParamDecorator({ value: (action: Action) => { const token = (action.request.headers.authorization || "").replace('Bearer ', '') || ''; const authService = Container.get(AuthService); const payload = authService.decodeToken(token); return payload; } }) } ================================================ FILE: aquila/search/src/helper/user.ts ================================================ import { Action } from "routing-controllers"; import Container from "typedi"; import { Customer } from "../entity/Customer"; import { CustomerTemp } from "../entity/CustomerTemp"; import { AuthService } from "../service/AuthService"; import { CustomerService } from "../service/CustomerService"; export async function currentUserChecker(action: Action): Promise { const token = (action.request.headers.authorization || "").replace('Bearer ', '') || ''; const authService = Container.get(AuthService); const customerService = Container.get(CustomerService); const payload = authService.decodeToken(token); const customer = await customerService.getCustomerById(payload.customerId, payload.accountStatus); return customer; } ================================================ FILE: aquila/search/src/index.ts ================================================ import { exit } from 'process'; import main from './app'; import { ConfigService } from './lib/ConfigService'; const configService = new ConfigService(); const port = configService.get('PORT'); main().then(app => { app.listen(port, () => console.log(`App running on port ${port}`)) }).catch(err => { console.log(err); exit(1); }) ================================================ FILE: aquila/search/src/job/AppQueue.ts ================================================ import { Job, Queue } from "bullmq"; import { Service } from "typedi"; import connection from '../config/redisConnection'; import { AppJobNames } from "./types"; @Service() export class AppQueue { public queue: Queue; public constructor(){ this.queue = new Queue("APP", { connection }) } public async add(name: AppJobNames, data: T) { return await this.queue.add(name, data); } } ================================================ FILE: aquila/search/src/job/appWorker.ts ================================================ import { Worker } from "bullmq"; import Redis from "ioredis"; export function getAppWorker(connection: Redis) { const processorFile = `${__dirname}/appWorkerProcessor.js`; const worker = new Worker('APP', processorFile, { connection }); return worker; } ================================================ FILE: aquila/search/src/job/appWorkerProcessor.ts ================================================ import { Job } from "bullmq"; import Parser from "@postlight/parser"; import axios from 'axios'; import Container from "typedi"; import { DataSource } from "typeorm"; import db from '../config/db'; import { AppJobData, AppJobNames, IndexDocumentData } from "./types"; import { AquilaClientService } from "../lib/AquilaClientService"; import { AccountStatus } from "../service/dto/AuthServiceDto"; import { CollectionTemp } from "../entity/CollectionTemp"; import { Collection } from "../entity/Collection"; import { Bookmark, BookmarkStatus } from "../entity/Bookmark"; import { BookmarkPara } from "../entity/BookmarkPara"; import { BookmarkParaTemp } from "../entity/BookmarkParaTemp"; import { BookmarkTemp, BookmarkTempStatus } from "../entity/BookmarkTemp"; import { ConfigService } from "../lib/ConfigService"; async function summarize(html: string) { const configService = Container.get(ConfigService); const response = await axios.post(configService.get("SUMMARIZER_URL"), { html }); return response.data.result; } let dataSource: DataSource; export default async function(job: Job) { if(job.name === AppJobNames.INDEX_DOCUMENT) { const { data } = <{data: IndexDocumentData}>job; // load database if(!dataSource) { dataSource = await db.initialize(); } // load bookmark let bookmarkObj: Bookmark | BookmarkTemp | null; if(data.accountStatus === AccountStatus.TEMPORARY) { bookmarkObj = await BookmarkTemp.findOne({ where: {id: data.bookmarkId}}); }else { bookmarkObj = await Bookmark.findOne({ where: {id : data.bookmarkId}}); } if(bookmarkObj) { const bookmark: Bookmark | BookmarkTemp = bookmarkObj; // extract metadata from html const parsedHtml = await Parser.parse(bookmark.url, { html: bookmark.html}); // generate array summary from text content const summary = await summarize(parsedHtml.content || ""); if(bookmark.title) { summary.push(bookmark.title); } if(bookmark.summary) { summary.push(bookmark.summary); } // connect to aquila const aquilaClient = Container.get(AquilaClientService) await aquilaClient.connect(); // load collection let collection: Collection | CollectionTemp | null; if(data.accountStatus === AccountStatus.TEMPORARY) { collection = await CollectionTemp.findOne({ where: { id: bookmark.collectionId }}); }else { collection = await Collection.findOne({ where: { id: bookmark.collectionId }}); } if(collection === null) { throw new Error('Collection not found'); } // generate vector from array of paragraph const vectorArray = await aquilaClient.getHubServer().compressDocument(collection.aquilaDbName, summary); // bulk insert into aquiladb let bookmarkParas: BookmarkPara[] | BookmarkParaTemp[]; await db.transaction(async transactionalEntityManager => { // insert all para to bookmark_para table if(data.accountStatus === AccountStatus.TEMPORARY) { bookmarkParas = summary.map((para: string) => { const bookmarkPara = new BookmarkParaTemp() bookmarkPara.bookmarkId = bookmark.id, bookmarkPara.content = para; return bookmarkPara; }) await transactionalEntityManager.save(bookmarkParas); }else { bookmarkParas = summary.map((para: string) => { const bookmarkPara = new BookmarkPara() bookmarkPara.bookmarkId = bookmark.id, bookmarkPara.content = para; return bookmarkPara; }) await transactionalEntityManager.save(bookmarkParas); } // create documents on aquiladb const documents = summary.map((para: string, index: number) => { return { metadata: { para, bookmark_para_id: bookmarkParas[index].id, bookmark_id: bookmark.id }, code: vectorArray[index] } }) await aquilaClient.getDbServer().createDocuments((collection as Collection | CollectionTemp).aquilaDbName, documents) // update status as PROCESSED if(data.accountStatus === AccountStatus.TEMPORARY) { bookmark.status = BookmarkTempStatus.PROCESSED; }else { bookmark.status = BookmarkStatus.PROCESSED; } await transactionalEntityManager.save(bookmark); }); } } } ================================================ FILE: aquila/search/src/job/types.ts ================================================ import { Bookmark } from "../entity/Bookmark" import { BookmarkTemp } from "../entity/BookmarkTemp" import { AccountStatus } from "../service/dto/AuthServiceDto"; export enum AppJobNames { INDEX_DOCUMENT = 'INDEX_DOCUMENT' } export interface IndexDocumentData { bookmarkId: string; accountStatus: AccountStatus } export interface AddData { name: string; } export type AppJobData = IndexDocumentData; ================================================ FILE: aquila/search/src/lib/AquilaClientService.ts ================================================ import { AquilaClient, Db, Hub, Schema, Wallet } from "aquila-js"; import path from "path"; import { Service } from "typedi"; import crypto from 'crypto'; import { ConfigService } from "./ConfigService"; @Service() export class AquilaClientService { private db: Db; private hub: Hub; public constructor(private configService: ConfigService) {} public async connect() { const wallet = new Wallet( path.join(this.configService.get("AQUILA_DB_KEY_PATH"))); const dbUrl = this.configService.get("AQUILA_DB_HOST"); const dbPort = parseInt(this.configService.get("AQUILA_DB_PORT"), 10); const hubWallet = new Wallet( path.join(this.configService.get("AQUILA_HUB_KEY_PATH"))); const hubUrl = this.configService.get("AQUILA_HUB_HOST"); const hubPort = parseInt(this.configService.get("AQUILA_HUB_PORT"), 10); // connecting to aquila db server this.db = await AquilaClient.getDbServer(dbUrl, dbPort, wallet); // connecting to aquila hub server this.hub = await AquilaClient.getHubServer(hubUrl, hubPort, hubWallet); } public getDbServer() { return this.db; } public getHubServer() { return this.hub; } public async createCollection(desc: string, secretKey: string) { const hashSecret = crypto.createHash('sha256'); const schema: Schema = { description: desc, unique: hashSecret.update(secretKey).digest('hex'), encoder: "strn:msmarco-distilbert-base-tas-b", codelen: 768, metadata: { "para": "string", "bookmark_para_id": "string", "bookmark_id": "string" } }; await this.hub.createDatabase(schema); const dbName = await this.db.createDatabase(schema); return dbName; } } ================================================ FILE: aquila/search/src/lib/ConfigService.ts ================================================ import fs from 'fs'; import dotenv from 'dotenv'; import { Service } from 'typedi'; @Service() export class ConfigService { constructor() { this.loadEnv(); } private loadEnv() { let envFile; const env = process.env.NODE_ENV || 'development'; switch (env) { case 'production': envFile = '.env.production'; break; case 'test': envFile = '.env.test'; break; default: envFile = '.env.development'; } const path = `${__dirname}/../../${envFile}`; const options: dotenv.DotenvConfigOptions = {} if(fs.existsSync(path)) { options.path = path; } dotenv.config(options); } get(key: string, defaultValue?: string): string { let data = process.env[key]; if (!data && !defaultValue) { throw new Error(`Environment variable not found [${key}]`); } if (!data && defaultValue) { data = defaultValue; } return data as string; } } ================================================ FILE: aquila/search/src/middleware/global/AuthMiddleware.ts ================================================ import { NextFunction, Request, Response } from 'express'; import { ExpressMiddlewareInterface } from 'routing-controllers'; import { Service } from 'typedi'; import { AuthService } from '../../service/AuthService'; @Service() export class AuthMiddleware implements ExpressMiddlewareInterface { public constructor(private authService: AuthService) {} public use(req: Request, res: Response, next: NextFunction): void { const token = (req.headers.authorization || "").replace('Bearer ', '') || ''; this.authService.authenticate(token); next(); } } ================================================ FILE: aquila/search/src/middleware/global/GlobalErrorMiddleware.ts ================================================ import { NextFunction, Request, Response } from "express"; import { ExpressErrorMiddlewareInterface, HttpError, Middleware } from "routing-controllers"; import { Service } from "typedi"; import { ValidationError } from "../../utils/errors/ValidationError"; @Service() @Middleware({ type: 'after', priority: 1}) export class GlobalErrorHandler implements ExpressErrorMiddlewareInterface { error(err: Error, req: Request, res: Response, next: NextFunction) { console.log(err); if( err instanceof ValidationError) { return res.status(err.httpCode).send({ code: err.httpCode, name: err.name, message: err.message, errors: err.errors }); } if (err instanceof HttpError) { return res.status(err.httpCode).send({ code: err.httpCode, name: err.name, message: err.message, }); } else { const env = process.env.NODE_ENV; res.status(500).send({ code: 500, name: env === 'development' ? err.name : 'Unknown Error', message: env === 'development' ? err.message : 'Something went wrong', stack: env === 'development' ? err.stack : null, }); } next(); } } ================================================ FILE: aquila/search/src/middleware/global/TokenParserMiddleware.ts ================================================ import { NextFunction, Request, Response } from "express"; import { ExpressMiddlewareInterface, Middleware } from "routing-controllers"; import { Service } from "typedi"; import { AuthService } from "../../service/AuthService"; @Service() @Middleware({type: 'before'}) export class TokenParserMiddleware implements ExpressMiddlewareInterface { public constructor(private authService: AuthService) {} public use(req: Request, res: Response, next: NextFunction) { const token = (req.headers.authorization || "").replace('Bearer ', '') || ''; if(token) { req.token = token; } try{ const payload = this.authService.decodeToken(token); req.jwtTokenPayload = payload; }catch(e) {} next(); } } ================================================ FILE: aquila/search/src/middleware/validator/bookmark/AddBookmarkValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { body } from "express-validator"; import { ExpressMiddlewareInterface, Middleware } from "routing-controllers"; import Container, { Service } from "typedi"; import { JwtPayloadData } from "../../../helper/decorators/jwtPayloadData"; import { CollectionService } from "../../../service/CollectionService"; import { JwtPayload } from "../../../service/dto/AuthServiceDto"; import { validate } from "../../../utils/validate"; @Service() export class AddBookmarkValidator implements ExpressMiddlewareInterface { public constructor(private collectionService: CollectionService, @JwtPayloadData() private jwtPayloadData: JwtPayload) {} public async use(req: Request, res: Response, next: NextFunction) { const validators = [ body('html') // .trim().not().isEmpty() // .withMessage("Html is required") // .bail() .isLength({max: 900000}) .withMessage("Html content length should be less than 900,000 characters"), body("url") .trim().not().isEmpty() .withMessage("Url is required") .bail() .isLength({max: 2048}) .withMessage("Url must be less than 2048 characters") .bail() .isURL() .withMessage("Invalid url") .bail() .custom((value) => { const url = new URL(value); if(['www.localhost', 'localhost', 'www.127.0.0.1', '127.0.0.1'].includes(url.hostname)) { throw new Error(`Url '${value}' not allowed`) } return true; }), body('collectionId') .trim().not().isEmpty() .withMessage("Collection Id is required") .bail() .isUUID() .withMessage("Invalid Collection Id") .bail() .custom(async (value) => { const jwtPayloadData = req.jwtTokenPayload; if(jwtPayloadData) { const collectionService = Container.get(CollectionService); const collection = await collectionService.getCollectionById(value, jwtPayloadData.accountStatus); if(collection.customerId !== jwtPayloadData.customerId) { throw new Error("Invalid collection id"); } } return true; }) ] await validate(validators, req); next(); } } ================================================ FILE: aquila/search/src/middleware/validator/bookmark/GetBookmarkByCollectionIdValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { param, query } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import Container, { Service } from "typedi"; import { CollectionService } from "../../../service/CollectionService"; import { validate } from "../../../utils/validate"; @Service() export class GetBookmarkByCollectionIdValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ query("limit") .optional() .isNumeric() .withMessage("Limit should be number") .isInt({ min: 1}) .withMessage("Limit should be greater than or equal to 1") .isInt({ max: 10 }) .withMessage("Limit should be less than or equal to 10"), query("page") .optional() .isNumeric() .withMessage("Page should be number") .isInt({ min: 1}) .withMessage("Page should be greater than or equal to 1"), query("query") .optional() .isLength({ max: 50}) .withMessage("Query must be less than 50 characters"), param("collectionId") .isUUID() .withMessage("Invalid collection") .bail() .custom(async (value) => { const jwtPayloadData = req.jwtTokenPayload; if(jwtPayloadData) { const collectionService = Container.get(CollectionService); const collection = await collectionService.getCollectionById(value, jwtPayloadData.accountStatus); if(collection.customerId !== jwtPayloadData.customerId) { throw new Error("Invalid collection id"); } } return true; }) ]; await validate(validators, req) next(); } } ================================================ FILE: aquila/search/src/middleware/validator/bookmark/GetFeaturedBookmarkValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { query } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import { Service } from "typedi"; import { validate } from "../../../utils/validate"; @Service() export class GetFeaturedBookmarkValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ query("limit") .optional() .isNumeric() .withMessage("Limit should be number") .isInt({ min: 1}) .withMessage("Limit should be greater than or equal to 1") .isInt({ max: 10 }) .withMessage("Limit should be less than or equal to 10"), query("page") .optional() .isNumeric() .withMessage("Page should be number") .isInt({ min: 1}) .withMessage("Page should be greater than or equal to 1") ]; await validate(validators, req) next(); } } ================================================ FILE: aquila/search/src/middleware/validator/bookmark/GetPublicBookmarkByCollectionIdParamValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { param, query } from "express-validator"; import { ExpressMiddlewareInterface, NotFoundError } from "routing-controllers"; import Container, { Service } from "typedi"; import { CollectionService } from "../../../service/CollectionService"; import { AccountStatus } from "../../../service/dto/AuthServiceDto"; import { validate } from "../../../utils/validate"; @Service() export class GetPublicBookmarkByCollectionIdParamValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const collectionService = Container.get(CollectionService); const collection = await collectionService.getCollectionById(req.params.customerId, AccountStatus.PERMANENT); if(!collection.isShareable) { throw new NotFoundError("Collection Not found"); } next(); } } ================================================ FILE: aquila/search/src/middleware/validator/bookmark/GetPublicBookmarkByCollectionIdValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { param, query } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import Container, { Service } from "typedi"; import { CollectionService } from "../../../service/CollectionService"; import { AccountStatus } from "../../../service/dto/AuthServiceDto"; import { validate } from "../../../utils/validate"; @Service() export class GetPublicBookmarkByCollectionIdValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ query("limit") .optional() .isNumeric() .withMessage("Limit should be number") .isInt({ min: 1}) .withMessage("Limit should be greater than or equal to 1") .isInt({ max: 10 }) .withMessage("Limit should be less than or equal to 10"), query("page") .optional() .isNumeric() .withMessage("Page should be number") .isInt({ min: 1}) .withMessage("Page should be greater than or equal to 1"), query("query") .optional() .isLength({ max: 50}) .withMessage("Query must be less than 50 characters"), param("collectionId") .isUUID() .withMessage("Invalid collection") ]; await validate(validators, req) next(); } } ================================================ FILE: aquila/search/src/middleware/validator/collection/GetAllFeaturedCollectionValidator.ts ================================================ import { NextFunction, Request } from "express"; import { query } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import { Service } from "typedi"; import { validate } from "../../../utils/validate"; @Service() export class GetAllFeaturedCollectionValidator implements ExpressMiddlewareInterface { async use(req: Request, res: Response, next: NextFunction) { const validators = [ query("limit") .optional() .isNumeric() .withMessage("Limit must be a number") .isInt({min: 1}) .withMessage("Limit must be greater than 1") .isInt({ max: 10 }) .withMessage("Limit must be less than or equal to 10"), query("page") .optional() .isNumeric() .withMessage("Page must be a number") ] await validate(validators, req); next(); } } ================================================ FILE: aquila/search/src/middleware/validator/collection/GetAllPublicCollectionValidator.ts ================================================ import { NextFunction, Request } from "express"; import { query } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import { Service } from "typedi"; import { validate } from "../../../utils/validate"; @Service() export class GetAllPublicCollectionValidator implements ExpressMiddlewareInterface { async use(req: Request, res: Response, next: NextFunction) { const validators = [ query("limit") .optional() .isNumeric() .withMessage("Limit must be a number") .isInt({min: 1}) .withMessage("Limit must be greater than 1") .isInt({ max: 10 }) .withMessage("Limit must be less than or equal to 10"), query("page") .optional() .isNumeric() .withMessage("Page must be a number") ] await validate(validators, req); next(); } } ================================================ FILE: aquila/search/src/middleware/validator/collection/GetPublicCollectionByIdValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { param } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import { Service } from "typedi"; import { validate } from "../../../utils/validate"; @Service() export class GetPublicCollectionByIdValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ param("collectionId") .isUUID() .withMessage("Invalid collection id"), ]; await validate(validators, req); next() } } ================================================ FILE: aquila/search/src/middleware/validator/csubscription/SubscribeCollectionValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { BadRequestError, ExpressMiddlewareInterface } from "routing-controllers"; import { Service } from "typedi"; import { JwtPayloadData } from "../../../helper/decorators/jwtPayloadData"; import { CollectionSubscriptionService } from "../../../service/CollectionSubscriptionService"; import { JwtPayload } from "../../../service/dto/AuthServiceDto"; @Service() export class SubscribeCollectionValidator implements ExpressMiddlewareInterface { public constructor(private collectionSubService: CollectionSubscriptionService, @JwtPayloadData() private jwtPayloadData: JwtPayload) {} public async use(req: Request, res: Response, next: NextFunction) { const jwtPayloadData = req.jwtTokenPayload; if(jwtPayloadData) { const status = await this.collectionSubService.isCollectionSubscribedByCustomer(req.params.collectionId, jwtPayloadData.customerId, jwtPayloadData.accountStatus); if(status) { throw new BadRequestError("Collection is already subscribed"); } const totalSubscriptions = await this.collectionSubService.getTotalCollectionSubscribedByCustomer(jwtPayloadData.customerId, jwtPayloadData.accountStatus); if(totalSubscriptions > 5) { throw new BadRequestError("You have reached maximum number of subscriptions"); } } next(); } } ================================================ FILE: aquila/search/src/middleware/validator/csubscription/UnSubscribeCollectionValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { param } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import { Service } from "typedi"; import { validate } from "../../../utils/validate"; @Service() export class UnSubscribeCollectionValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ param('collectionId') .trim().not().isEmpty() .withMessage("Collection Id is required") .bail() .isUUID() .withMessage("Invalid Collection Id") ]; await validate(validators, req); next(); } } ================================================ FILE: aquila/search/src/middleware/validator/customer/ActivateCustomerValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { body } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import Container, { Service } from "typedi"; import { CustomerService } from "../../../service/CustomerService"; import { validate } from "../../../utils/validate"; @Service() export class ActivateCustomerValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ body('firstName') .trim().not().isEmpty() .withMessage("First name is required") .isLength({ min: 3}) .withMessage("First should have atleast 3 characters") .isLength({ max: 15}) .withMessage("First should be less than or equal to 15 characters") .matches(/^[a-zA-Z]*[a-zA-Z]$/).withMessage("Description must contain only alpha numeric characters"), body('lastName') .trim().not().isEmpty() .withMessage("Last name is required") .isLength({ min: 3}) .withMessage("First should have atleast 3 characters") .isLength({ max: 15}) .withMessage("First should be less than or equal to 15 characters") .matches(/^[a-zA-Z]*[a-zA-Z]$/).withMessage("Description must contain only alpha numeric characters"), body('email') .trim().not().isEmpty() .withMessage("Email is required") .isEmail() .withMessage("Invalid email") .bail() .custom(async (value) => { const customerService = Container.get(CustomerService); const customer = await customerService.findCustomerByEmailId(value); if(customer) { throw new Error("Email already exists"); } }), body('desc') .trim().not().isEmpty() .withMessage("Description is required") .isLength({ min: 50}) .withMessage("Description should have atleast 50 characters") .isLength({max: 255}) .withMessage("Description should be less than or equal to 255 characters"), // .matches(/^[a-zA-Z0-9]*[a-zA-Z0-9]$/).withMessage("Description must contain only alpha numeric characters"), ]; await validate(validators, req); next(); } } ================================================ FILE: aquila/search/src/middleware/validator/customer/CreateCustomerValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { body } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import { Service } from "typedi"; import { validate } from "../../../utils/validate"; @Service() export class CreateCustomerValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ body('firstName') .trim().not().isEmpty() .withMessage("First name is required") .isLength({ min: 3}) .withMessage("First should have atleast 3 characters") .isLength({ max: 15}) .withMessage("First should be less than or equal to 15 characters") .matches(/^[a-zA-Z]*[a-zA-Z]$/).withMessage("First name must contain only alpha characters"), body('lastName') .trim().not().isEmpty() .withMessage("Last name is required") .isLength({ min: 3}) .withMessage("First should have atleast 3 characters") .isLength({ max: 15}) .withMessage("First should be less than or equal to 15 characters") .matches(/^[a-zA-Z]*[a-zA-Z]$/).withMessage("Last name must contain only alpha characters"), ]; await validate(validators, req); next(); } } ================================================ FILE: aquila/search/src/middleware/validator/customer/GetCustomerPublicInfoByIdValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { param } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import { Service } from "typedi"; import { validate } from "../../../utils/validate"; @Service() export class GetCustomerPublicInfoByIdValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ param('customerId') .isUUID() .withMessage("Invalid Customer Id") ]; await validate(validators, req); next(); } } ================================================ FILE: aquila/search/src/middleware/validator/customer/UpdateCustomerValidator.ts ================================================ import { NextFunction, Request, Response } from "express"; import { body } from "express-validator"; import { ExpressMiddlewareInterface } from "routing-controllers"; import Container, { Service } from "typedi"; import { Customer } from "../../../entity/Customer"; import { CustomerService } from "../../../service/CustomerService"; import { AccountStatus } from "../../../service/dto/AuthServiceDto"; import { validate } from "../../../utils/validate"; @Service() export class UpdateCustomerValidator implements ExpressMiddlewareInterface { public async use(req: Request, res: Response, next: NextFunction) { const validators = [ body('firstName') .trim().not().isEmpty() .withMessage("First name is required") .isLength({ min: 3}) .withMessage("First should have atleast 3 characters") .isLength({ max: 15}) .withMessage("First should be less than or equal to 15 characters") .matches(/^[a-zA-Z]*[a-zA-Z]$/).withMessage("Description must contain only alpha numeric characters"), body('lastName') .trim().not().isEmpty() .withMessage("Last name is required") .isLength({ min: 3}) .withMessage("First should have atleast 3 characters") .isLength({ max: 15}) .withMessage("First should be less than or equal to 15 characters") .matches(/^[a-zA-Z]*[a-zA-Z]$/).withMessage("Description must contain only alpha numeric characters"), body('email') .trim().not().isEmpty() .withMessage("Email is required") .isEmail() .withMessage("Invalid email") .bail() .custom(async (value, meta) => { const jwtPayloadData = req.jwtTokenPayload; if(jwtPayloadData) { const customerService = Container.get(CustomerService); const customer = await customerService.getCustomerById(jwtPayloadData?.customerId, AccountStatus.PERMANENT) as Customer; if(value === customer.email) { return; } // check email exists const customerExists = await customerService.findCustomerByEmailId(value); if(customerExists) { throw new Error("Email already exists"); } } }), body('desc') .trim().not().isEmpty() .withMessage("Description is required") .isLength({ min: 50}) .withMessage("Description should have atleast 50 characters") .isLength({max: 255}) .withMessage("Description should be less than or equal to 255 characters") // .matches(/^[a-zA-Z0-9]*[a-zA-Z0-9]$/).withMessage("Description must contain only alpha numeric characters"), ]; await validate(validators, req); next(); } } ================================================ FILE: aquila/search/src/migrations/1676714378990-bookmark.ts ================================================ import { MigrationInterface, QueryRunner } from "typeorm"; export class bookmark1676714378990 implements MigrationInterface { name = 'bookmark1676714378990' public async up(queryRunner: QueryRunner): Promise { await queryRunner.query(`CREATE TYPE "public"."bookmark_status_enum" AS ENUM('NOT_PROCESSED', 'PROCESSING', 'PROCESSED')`); await queryRunner.query(`CREATE TABLE "bookmark" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "collection_id" uuid NOT NULL, "url" character varying(2048) NOT NULL, "html" text NOT NULL, "title" character varying(2048), "author" character varying(100), "cover_img" character varying(2048), "summary" text, "links" character varying(2048), "is_hidden" boolean NOT NULL DEFAULT false, "status" "public"."bookmark_status_enum" NOT NULL DEFAULT 'NOT_PROCESSED', "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_b7fbf4a865ba38a590bb9239814" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "bookmark_para_temp" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "bookmark_id" character varying NOT NULL, "content" text NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_96228babfbf0e3d39503847b19f" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "bookmark_para" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "bookmark_id" character varying NOT NULL, "content" text NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_80453e30c727ef75d8d3a227ae1" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TYPE "public"."bookmark_temp_status_enum" AS ENUM('NOT_PROCESSED', 'PROCESSING', 'PROCESSED')`); await queryRunner.query(`CREATE TABLE "bookmark_temp" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "collection_id" uuid NOT NULL, "url" character varying(2048) NOT NULL, "html" text NOT NULL, "title" character varying(100), "author" character varying(100), "cover_img" character varying(255), "summary" character varying(255), "links" character varying(255), "is_hidden" boolean NOT NULL DEFAULT false, "status" "public"."bookmark_temp_status_enum" NOT NULL DEFAULT 'NOT_PROCESSED', "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_1e23a591867b46d1d25cdd69421" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "customer" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "customer_id" BIGSERIAL NOT NULL, "first_name" character varying(20) NOT NULL, "last_name" character varying(20) NOT NULL, "avatar" character varying(255) NOT NULL, "email" character varying(50) NOT NULL, "desc" character varying(255) NOT NULL, "secret_key" character varying(255) NOT NULL, "lightning_address" character varying(255), "is_active" boolean NOT NULL DEFAULT true, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_5f387140f14dda0d07e8c69d94e" UNIQUE ("secret_key"), CONSTRAINT "UQ_0eb8dbfdf208da3611520781da9" UNIQUE ("lightning_address"), CONSTRAINT "PK_a7a13f4cacb744524e44dfdad32" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "collection" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying(25) NOT NULL, "desc" character varying(255) NOT NULL, "customer_id" uuid NOT NULL, "aquila_db_name" character varying(255) NOT NULL, "is_shareable" boolean NOT NULL DEFAULT true, "indexed_docs_count" bigint NOT NULL DEFAULT '0', "is_featured" boolean NOT NULL DEFAULT false, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_f6f5b901feb7458d7f0ae8e40a0" UNIQUE ("aquila_db_name"), CONSTRAINT "PK_ad3f485bbc99d875491f44d7c85" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "collection_subscription" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "collection_id" uuid NOT NULL, "subscriber_id" uuid NOT NULL, "subscribed_at" TIMESTAMP NOT NULL DEFAULT now(), "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_871ce8ad8c0bd303596085c69b3" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "collection_subscription_temp" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "collection_id" uuid NOT NULL, "subscriber_id" uuid NOT NULL, "subscribed_at" TIMESTAMP NOT NULL DEFAULT now(), "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_0c00e1823a7935706679daab419" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "collection_temp" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying(25) NOT NULL, "desc" character varying(255) NOT NULL, "customer_id" uuid NOT NULL, "aquila_db_name" character varying(255) NOT NULL, "is_shareable" boolean NOT NULL DEFAULT true, "indexed_docs_count" bigint NOT NULL DEFAULT '0', "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_55a8fd150df3f170c789a81affe" UNIQUE ("aquila_db_name"), CONSTRAINT "PK_658841bdb83ea4263af2e64a683" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "customer_temp" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "customer_id" BIGSERIAL NOT NULL, "first_name" character varying(20) NOT NULL, "last_name" character varying(20) NOT NULL, "avatar" character varying(255) NOT NULL, "desc" character varying(255) NOT NULL, "secret_key" character varying(255) NOT NULL, "is_active" boolean NOT NULL DEFAULT true, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_0754f469522cf14ad9beae89a61" UNIQUE ("secret_key"), CONSTRAINT "PK_0a1542f39ffea6cffd26ce1a731" PRIMARY KEY ("id"))`); await queryRunner.query(`ALTER TABLE "collection" ADD CONSTRAINT "FK_fbcfcef552d9e814bc2980b4401" FOREIGN KEY ("customer_id") REFERENCES "customer"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); } public async down(queryRunner: QueryRunner): Promise { await queryRunner.query(`ALTER TABLE "collection" DROP CONSTRAINT "FK_fbcfcef552d9e814bc2980b4401"`); await queryRunner.query(`DROP TABLE "customer_temp"`); await queryRunner.query(`DROP TABLE "collection_temp"`); await queryRunner.query(`DROP TABLE "collection_subscription_temp"`); await queryRunner.query(`DROP TABLE "collection_subscription"`); await queryRunner.query(`DROP TABLE "collection"`); await queryRunner.query(`DROP TABLE "customer"`); await queryRunner.query(`DROP TABLE "bookmark_temp"`); await queryRunner.query(`DROP TYPE "public"."bookmark_temp_status_enum"`); await queryRunner.query(`DROP TABLE "bookmark_para"`); await queryRunner.query(`DROP TABLE "bookmark_para_temp"`); await queryRunner.query(`DROP TABLE "bookmark"`); await queryRunner.query(`DROP TYPE "public"."bookmark_status_enum"`); } } ================================================ FILE: aquila/search/src/service/AuthService.ts ================================================ import jwt, { Jwt } from 'jsonwebtoken'; import { BadRequestError, UnauthorizedError } from 'routing-controllers'; import { Service } from 'typedi'; import { Customer } from '../entity/Customer'; import { CustomerTemp } from '../entity/CustomerTemp'; import { ConfigService } from '../lib/ConfigService'; import { AccountStatus, JwtPayload } from './dto/AuthServiceDto'; @Service() export class AuthService { public constructor(private configService: ConfigService) {} public authenticate(token: string) { const secret = this.configService.get('JWT_SECRET'); try { jwt.verify(token, secret); }catch(e) { throw new UnauthorizedError("Authorization failed"); } } public decodeToken(token: string): JwtPayload { const decodedToken = jwt.decode(token) as JwtPayload; return decodedToken; } private generateToken(payload: JwtPayload): string { const jwtSecret = this.configService.get('JWT_SECRET'); const expiresIn = this.configService.get('JWT_EXPIRES_IN'); const token = jwt.sign(payload, jwtSecret, { expiresIn }) return token; } public async login(secretKey: string): Promise { try{ return await this.loginPermanentCustoemr(secretKey); }catch(e) { if(!(e instanceof BadRequestError)) { throw e; } } return await this.loginTemporaryCustomer(secretKey); } public async loginPermanentCustoemr(secretKey: string): Promise{ // check secret is valid const customer = await Customer.findOne({ where: {secretKey} }) if(!customer) { throw new BadRequestError("Invalid credentials"); } // if valid secret generate token const payload = { customerId: customer.id, firstName: customer.firstName, lastName: customer.lastName, createdAt: customer.createdAt, accountStatus: AccountStatus.PERMANENT }; const token = this.generateToken(payload); return token; } public async loginTemporaryCustomer(secretKey: string): Promise { // check secret is valid const customer = await CustomerTemp.findOne({ where: { secretKey } }) if(!customer) { throw new BadRequestError("Invalid credentials"); } // if valid secret generate token const payload = { customerId: customer.id, firstName: customer.firstName, lastName: customer.lastName, accountStatus: AccountStatus.TEMPORARY }; const token = this.generateToken(payload); return token; } } ================================================ FILE: aquila/search/src/service/BookmarkService.ts ================================================ import { BadRequestError, InternalServerError } from "routing-controllers"; import { Service } from "typedi"; import { In } from "typeorm"; import puppeteer from 'puppeteer-core'; import * as cheerio from "cheerio"; import dataSource from "../config/db"; import { Bookmark, BookmarkStatus } from "../entity/Bookmark"; import { BookmarkPara } from "../entity/BookmarkPara"; import { BookmarkParaTemp } from "../entity/BookmarkParaTemp"; import { BookmarkTemp, BookmarkTempStatus } from "../entity/BookmarkTemp"; import { Collection } from "../entity/Collection"; import { CollectionTemp } from "../entity/CollectionTemp"; import { AppQueue } from "../job/AppQueue"; import { AppJobNames, IndexDocumentData } from "../job/types"; import { AquilaClientService } from "../lib/AquilaClientService"; import { AccountStatus } from "./dto/AuthServiceDto"; import { AddBookmarkInputDto, GetAllBookmarksByCollectionIdOptionsInputDto, GetBookmarksByCollectionIdOutputDto, GetBookmarksByCollectionIdOptionsInputDto, GetFeaturedBookmarksOutputDto } from "./dto/BookmarkServiceDto"; import { ConfigService } from "../lib/ConfigService"; @Service() export class BookmarkService { public constructor(private appQueue: AppQueue, private aquilaClientService: AquilaClientService, private configService: ConfigService) {} private async fetchWebsiteContent(url: string): Promise { const broswerlessAPIKey = this.configService.get("BROWSERLESS_API_KEY"); const browser = await puppeteer.connect({ browserWSEndpoint: `wss://chrome.browserless.io?token=${broswerlessAPIKey}&ignoreDefaultArgs=true`, }); const page = await browser.newPage(); await page.goto(url, {waitUntil: 'domcontentloaded'}); const content = await page.content(); await browser.close(); return content; } private async addTemporaryBookmark(data: AddBookmarkInputDto): Promise { let bookmark = new BookmarkTemp(); await dataSource.transaction(async transactionalEntityManager => { bookmark.url = data.url; let htmlTmp = "" if(data.html === undefined) { htmlTmp = await this.fetchWebsiteContent(data.url); } const $ = cheerio.load(htmlTmp); const title = $("head").find("title").text() || ""; const metaDescription = $("meta[name='description']").attr("content") || ""; bookmark.html = data.html || htmlTmp; bookmark.title = title; bookmark.summary = metaDescription; bookmark.collectionId = data.collectionId; await transactionalEntityManager.save(bookmark); await this.appQueue.add(AppJobNames.INDEX_DOCUMENT, { accountStatus: AccountStatus.TEMPORARY, bookmarkId: bookmark.id}); }) return bookmark; } private async addPermanentBookmark(data: AddBookmarkInputDto): Promise { let bookmark = new Bookmark(); await dataSource.transaction(async transactionalEntityManager => { bookmark.url = data.url; let htmlTmp = "" if(data.html === undefined) { htmlTmp = await this.fetchWebsiteContent(data.url); } const $ = cheerio.load(htmlTmp); const title = $("head").find("title").text() || ""; const metaDescription = $("meta[name='description']").attr("content") || ""; bookmark.html = data.html || htmlTmp; bookmark.title = title; bookmark.summary = metaDescription; bookmark.collectionId = data.collectionId; await transactionalEntityManager.save(bookmark); await this.appQueue.add(AppJobNames.INDEX_DOCUMENT, { accountStatus: AccountStatus.PERMANENT, bookmarkId: bookmark.id}); }) return bookmark; } public async addBookmark(data: AddBookmarkInputDto, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.addTemporaryBookmark(data); } return await this.addPermanentBookmark(data); } private async getAllTemporaryBookmarksByCollectionId(collectionId: string, options: GetAllBookmarksByCollectionIdOptionsInputDto): Promise { const skip = (options.page - 1) * options.limit; const take = options.limit; const totalRecords = await BookmarkTemp.count({ where: { collectionId, status: BookmarkTempStatus.PROCESSED }}); const bookmarks = await BookmarkTemp.find({ where: { collectionId, status: BookmarkTempStatus.PROCESSED }, skip, take }); // find all paragraphs for bookmark const bookmarkIds = bookmarks.map(item => item.id); const paras = await BookmarkParaTemp.find({ where: { bookmarkId: In(bookmarkIds)}}); const bookmarkData = bookmarks.map(item => ({ id: item.id, collectionId: item.collectionId, url: item.url, title: item.title, author: item.author, coverImg: item.coverImg, summary: item.summary, description: paras.find(para => para.bookmarkId === item.id)?.content || '', createdAt: item.createdAt })); return { totalRecords, totalPages: Math.ceil(totalRecords / take), currentPage: options.page, limit: options.limit, bookmarks: bookmarkData } } private async getAllPermanentBookmarksByCollectionId(collectionId: string, options: GetAllBookmarksByCollectionIdOptionsInputDto): Promise { const skip = (options.page - 1) * options.limit; const take = options.limit; const totalRecords = await Bookmark.count({ where: { collectionId, status: BookmarkStatus.PROCESSED }}); const bookmarks = await Bookmark.find({ where: { collectionId, status: BookmarkStatus.PROCESSED}, skip, take }); // find all paragraphs for bookmark const bookmarkIds = bookmarks.map(item => item.id); const paras = await BookmarkPara.find({ where: { bookmarkId: In(bookmarkIds)}}); const bookmarkData = bookmarks.map(item => ({ id: item.id, collectionId: item.collectionId, url: item.url, title: item.title, author: item.author, coverImg: item.coverImg, summary: item.summary, description: paras.find(para => para.bookmarkId === item.id)?.content || '', createdAt: item.createdAt })); return { totalRecords, totalPages: Math.ceil(totalRecords / take), currentPage: options.page, limit: options.limit, bookmarks: bookmarkData } } private async getTemporaryBookmarksByCollectionId(collectionId: string, options: GetBookmarksByCollectionIdOptionsInputDto): Promise { if(!options.query) { return await this.getAllTemporaryBookmarksByCollectionId(collectionId, options); } const collection = await CollectionTemp.findOne({ where: { id: collectionId }}); if(!collection) { throw new BadRequestError("Invalid collection id"); } // generate vector from hub for the search query const vector = await this.aquilaClientService.getHubServer().compressDocument(collection.aquilaDbName, [options.query]) as number[][]; if(vector.length === 0) { throw new InternalServerError("Something went wrong"); } // search on aquiladb const { docs, dists } = await this.aquilaClientService.getDbServer().searchKDocuments(collection.aquilaDbName, vector, 100) const documentObjs: any = {}; // const newDists = dists[0].map(dist => (1 - dist)) const newDists = dists[0]; docs[0].forEach((doc: any, index: number) => { const docExists = documentObjs[doc.metadata.bookmark_id]; if(!docExists) { documentObjs[doc.metadata.bookmark_id] = { bookmarkParaId: doc.metadata.bookmark_para_id, bookmarkId: doc.metadata.bookmark_id, dist: newDists[index], paras: [{ dist: newDists[index], para: doc.metadata.para}] } } }); const documents = Object.values(documentObjs).sort((a: any, b: any) => (b.dist - a.dist)) // sort result from aquiladb and select records within limit and offset const totalRecords = documents.length; const start = (options.page -1) * options.limit; const end = ((options.page -1) * options.limit) + options.limit; const records = documents.slice(start, end); // fetch all bookmarks const bookmarkIds = records.map((item: any) => item.bookmarkId); const bookmarks = await BookmarkTemp.find({ where: { id: In(bookmarkIds)}}); // generate result const bookmarkData = records.map((item: any) => { const bookmark = bookmarks.find(data => data.id === item.bookmarkId) as BookmarkTemp; return ({ id: bookmark.id, collectionId: bookmark.collectionId, url: bookmark.url, title: bookmark.title, author: bookmark.author, coverImg: bookmark.coverImg, summary: bookmark.summary, description: item.paras[0].para }); }); const output = { totalRecords, totalPages: Math.ceil(totalRecords / options.limit), currentPage: options.page, limit: options.limit, bookmarks: bookmarkData } return output; } private async getPermanentBookmarksByCollectionId(collectionId: string, options: GetBookmarksByCollectionIdOptionsInputDto): Promise { if(!options.query) { return await this.getAllPermanentBookmarksByCollectionId(collectionId, options); } const collection = await Collection.findOne({ where: { id: collectionId }}); if(!collection) { throw new BadRequestError("Invalid collection id"); } // generate vector from hub for the search query const vector = await this.aquilaClientService.getHubServer().compressDocument(collection.aquilaDbName, [options.query]) as number[][]; if(vector.length === 0) { throw new InternalServerError("Something went wrong"); } // search on aquiladb const { docs, dists } = await this.aquilaClientService.getDbServer().searchKDocuments(collection.aquilaDbName, vector, 100) const documentObjs: any = {}; // const newDists = dists[0].map(dist => (1 - dist)) const newDists = dists[0]; docs[0].forEach((doc: any, index: number) => { const docExists = documentObjs[doc.metadata.bookmark_id]; if(!docExists) { documentObjs[doc.metadata.bookmark_id] = { bookmarkParaId: doc.metadata.bookmark_para_id, bookmarkId: doc.metadata.bookmark_id, dist: newDists[index], paras: [{ dist: newDists[index], para: doc.metadata.para}] } } }); const documents = Object.values(documentObjs).sort((a: any, b: any) => (b.dist - a.dist)) // sort result from aquiladb and select records within limit and offset const totalRecords = documents.length; const start = (options.page -1) * options.limit; const end = ((options.page -1) * options.limit) + options.limit; const records = documents.slice(start, end); // fetch all bookmarks const bookmarkIds = records.map((item: any) => item.bookmarkId); const bookmarks = await Bookmark.find({ where: { id: In(bookmarkIds)}}); // generate result const bookmarkData = records.map((item: any) => { const bookmark = bookmarks.find(data => data.id === item.bookmarkId) as Bookmark; return ({ id: bookmark.id, collectionId: bookmark.collectionId, url: bookmark.url, title: bookmark.title, author: bookmark.author, coverImg: bookmark.coverImg, summary: bookmark.summary, description: item.paras[0].para }); }); const output = { totalRecords, totalPages: Math.ceil(totalRecords / options.limit), currentPage: options.page, limit: options.limit, bookmarks: bookmarkData } return output; } public async getBookmarksByCollectionId(collectionId: string, options: GetBookmarksByCollectionIdOptionsInputDto, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.getTemporaryBookmarksByCollectionId(collectionId, options); } return await this.getPermanentBookmarksByCollectionId(collectionId, options); } public async getFeaturedBookmarks(options: GetBookmarksByCollectionIdOptionsInputDto): Promise { const skip = (options.page - 1) * options.limit; const take = options.limit; const query = Bookmark.createQueryBuilder("bookmark") .innerJoinAndSelect(Collection, "collection", "collection.id = bookmark.collectionId") .where("collection.isFeatured = :isFeatured", { isFeatured: true }) // .orderBy("RANDOM()") const totalRecords = await query.getCount() const featuredBookmarks = await query.skip(skip).take(take).getMany() // find all paragraphs for bookmark const bookmarkIds = featuredBookmarks.map(item => item.id); const paras = await BookmarkPara.find({ where: { bookmarkId: In(bookmarkIds)}}); const bookmarkData = featuredBookmarks.map(item => ({ id: item.id, collectionId: item.collectionId, url: item.url, title: item.title, author: item.author, coverImg: item.coverImg, summary: item.summary, description: paras.find(para => para.bookmarkId === item.id)?.content || '' })); return { totalRecords, totalPages: Math.ceil(totalRecords / take), currentPage: options.page, limit: options.limit, bookmarks: bookmarkData } } } ================================================ FILE: aquila/search/src/service/CollectionService.ts ================================================ import { NotFoundError } from "routing-controllers"; import { Service } from "typedi"; import { Collection } from "../entity/Collection"; import { CollectionTemp } from "../entity/CollectionTemp"; import { AccountStatus } from "./dto/AuthServiceDto"; import { GetAllCollectionsInputOptionsDto, GetAllCollectionsOutputDto } from "./dto/CollectionServiceDto"; @Service() export class CollectionService { public constructor() {} private async getAllTemporaryCollections(options: GetAllCollectionsInputOptionsDto): Promise { const allowedWhere: ['isShareable', 'isFeatured'] = ['isShareable', 'isFeatured']; const where = allowedWhere.reduce((prev: {[key: string]: string | boolean}, current) =>{ if(options.where && current in options.where) { prev[current] = options.where[current] as boolean | string; } return prev; }, {}) const totalRecords = await CollectionTemp.count({ where }); const skip = (options.page - 1) * options.limit; const take = options.limit; const collections = await CollectionTemp.find({ where, order: { createdAt: "DESC" }, take, skip}); return { totalRecords, totalPages: Math.ceil(totalRecords / options.limit), currentPage: options.page, limit: options.limit, collections }; } public async getAllPermanentCollections(options: GetAllCollectionsInputOptionsDto): Promise { const allowedWhere: ['isShareable', 'isFeatured'] = ['isShareable', 'isFeatured']; const where = allowedWhere.reduce((prev: {[key: string]: string | boolean}, current) =>{ if(options.where && current in options.where) { prev[current] = options.where[current] as boolean | string; } return prev; }, {}) const totalRecords = await Collection.count({ where }); const skip = (options.page - 1) * options.limit; const take = options.limit; const collections = await Collection.find({ where, order: { createdAt: "DESC" }, take, skip, relations: { customer: true }}); return { totalRecords, totalPages: Math.ceil(totalRecords / options.limit), currentPage: options.page, limit: options.limit, collections }; } public async getAllCollections(options: GetAllCollectionsInputOptionsDto, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.getAllTemporaryCollections(options); } return await this.getAllPermanentCollections(options); } private async getTempCustomerCollectionsByCustomerId(customerId: string): Promise { const collections = await CollectionTemp.find({ where: { customerId }}); return collections; } private async getPermanentCustomerCollectionsByCustomerId(customerId: string): Promise { const collections = await Collection.find({ where: { customerId }}); return collections; } public async getCustomerCollectionsByCustomerId(customerId: string, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.PERMANENT) { return await this.getPermanentCustomerCollectionsByCustomerId(customerId); } return await this.getTempCustomerCollectionsByCustomerId(customerId); } public async getTemporaryCollectionById(id: string): Promise { const collection = await CollectionTemp.findOne({ where: { id } }); if(!collection) { throw new NotFoundError("Collection Not found"); } return collection; } public async getPermanentCollectionById(id: string): Promise { const collection = await Collection.findOne({ where: { id }}); if(!collection) { throw new NotFoundError("Collection not found"); } return collection; } public async getCollectionById(id: string, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.getTemporaryCollectionById(id); } return await this.getPermanentCollectionById(id) } public async getPublicCollectionById(id: string): Promise { const collection = await Collection.findOne({ where: { id, isShareable: true }}); if(!collection) { throw new NotFoundError("Collection not found"); } return collection; } } ================================================ FILE: aquila/search/src/service/CollectionSubscriptionService.ts ================================================ import { InternalServerError, NotFoundError } from "routing-controllers"; import { Service } from "typedi"; import { In } from "typeorm"; import dataSource from "../config/db"; import { Bookmark, BookmarkStatus } from "../entity/Bookmark"; import { BookmarkPara } from "../entity/BookmarkPara"; import { Collection } from "../entity/Collection"; import { CollectionSubscription } from "../entity/CollectionSubscription"; import { CollectionSubscriptionTemp } from "../entity/CollectionSubscriptionTemp"; import { AquilaClientService } from "../lib/AquilaClientService"; import { AccountStatus } from "./dto/AuthServiceDto"; import { GetSubscriptionsByCustomerIdOptionsInputDto, GetSubscriptionsByCustomerIdOutputDto } from "./dto/CollectionSubscriptionServiceDto"; @Service() export class CollectionSubscriptionService { public constructor(private aquilaClientService: AquilaClientService) {} private async subscribeTemporaryCollection(collectionId: string, customerId: string): Promise { let subscription = new CollectionSubscriptionTemp(); await dataSource.transaction(async transactionalEntityManager => { subscription.collectionId = collectionId; subscription.subscriberId = customerId; await transactionalEntityManager.save(subscription); }) return subscription; } private async subscribePermanentCollection(collectionId: string, customerId: string): Promise { let subscription = new CollectionSubscription(); await dataSource.transaction(async transactionalEntityManager => { subscription.collectionId = collectionId; subscription.subscriberId = customerId; await transactionalEntityManager.save(subscription); }) return subscription; } public async subscribeCollection(collectionId: string, customerId: string, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.subscribeTemporaryCollection(collectionId, customerId); } return await this.subscribePermanentCollection(collectionId, customerId); } private async getTemporaryCollectionSubscriptionList(customerId: string): Promise { const collections = await CollectionSubscriptionTemp.find({ where: { subscriberId: customerId }}); return collections; } private async getPermanentCollectionSubscriptionList(customerId: string): Promise { const collections = await CollectionSubscription.find({ where: { subscriberId: customerId }}); return collections; } public async getCustomerSubscriptions(customerId: string, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.getTemporaryCollectionSubscriptionList(customerId); } return await this.getPermanentCollectionSubscriptionList(customerId); } public async isCollectionSubscribedByTemporaryCustomer(collectionId: string, customerId: string): Promise { // will return null if query can't find a result const subscription = await CollectionSubscriptionTemp.findOne({ where: { collectionId, subscriberId: customerId} }); if(subscription) { return true; } return false; } public async isCollectionSubscribedByPermanentCustomer(collectionId: string, customerId: string): Promise { // will return null if query can't find a result const subscription = await CollectionSubscription.findOne({ where: { collectionId, subscriberId: customerId} }); if(subscription) { return true; } return false; } public async isCollectionSubscribedByCustomer(collectionId: string, customerId: string, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.isCollectionSubscribedByTemporaryCustomer(collectionId, customerId); } return await this.isCollectionSubscribedByPermanentCustomer(collectionId, customerId); } private async unSubscribeTemporaryCollection(collectionId: string, customerId: string): Promise { const collection = await CollectionSubscriptionTemp.findOne({ where: { collectionId, subscriberId: customerId }}); if(!collection) { throw new NotFoundError("Subscription not found"); } await collection.remove(); return collection; } private async unSubscribePermanentCollection(collectionId: string, customerId: string): Promise { const collection = await CollectionSubscription.findOne({ where: { collectionId, subscriberId: customerId }}); if(!collection) { throw new NotFoundError("Subscription not found"); } await collection.remove() return collection; } public async unSubscribeCollection(collectionId: string, customerId: string, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.unSubscribeTemporaryCollection(collectionId, customerId); } return await this.unSubscribePermanentCollection(collectionId, customerId); } private async getTemporaryCollectionFollowerCount(collectionId: string): Promise { const count = await CollectionSubscriptionTemp.count({ where: { collectionId }}); return count; } private async getPermanentCollectionFollowerCount(collectionId: string): Promise { const count = await CollectionSubscription.count({ where: { collectionId }}); return count; } public async getCollectionSubscriberCount(collectionId: string, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return await this.getTemporaryCollectionFollowerCount(collectionId); } return await this.getPermanentCollectionFollowerCount(collectionId); } public async getTotalCollectionSubscribedByTemporaryCustomer(subscriberId: string): Promise { const count = await CollectionSubscriptionTemp.count({ where: { subscriberId }}); return count; } public async getTotalCollectionSubscribedByPermanentCustomer(subscriberId: string): Promise { const count = await CollectionSubscription.count({ where: { subscriberId }}); return count; } public async getTotalCollectionSubscribedByCustomer(customerId: string, accountStatus: AccountStatus): Promise { if(accountStatus === AccountStatus.TEMPORARY) { return this.getTotalCollectionSubscribedByTemporaryCustomer(customerId); } return this.getTotalCollectionSubscribedByPermanentCustomer(customerId); } private async getAllSubscriptionsByCustomerId(collectionIds: string[], options: GetSubscriptionsByCustomerIdOptionsInputDto): Promise { const skip = (options.page - 1) * options.limit; const take = options.limit; const totalRecords = await Bookmark.count({ where: { collectionId: In(collectionIds), status: BookmarkStatus.PROCESSED }}); const bookmarks = await Bookmark.find({ where: { collectionId: In(collectionIds), status: BookmarkStatus.PROCESSED}, skip, take }); // find all paragraphs for bookmark const bookmarkIds = bookmarks.map(item => item.id); const paras = await BookmarkPara.find({ where: { bookmarkId: In(bookmarkIds)}}); const bookmarkData = bookmarks.map(item => ({ id: item.id, collectionId: item.collectionId, url: item.url, title: item.title, author: item.author, coverImg: item.coverImg, summary: item.summary, description: paras.find(para => para.bookmarkId === item.id)?.content || '', createdAt: item.createdAt })); return { totalRecords, totalPages: Math.ceil(totalRecords / take), currentPage: options.page, limit: options.limit, bookmarks: bookmarkData } } public async getSubscriptionsByTemporaryCustomerId(customerId: string, options: GetSubscriptionsByCustomerIdOptionsInputDto): Promise { const collectionSubscriptions = await CollectionSubscriptionTemp.find({ where: { subscriberId: customerId }}); const collectionIds = collectionSubscriptions.map(collection => collection.collectionId); if(!options.query) { return this.getAllSubscriptionsByCustomerId(collectionIds, options); } const collections = await Collection.find({ where : { id: In(collectionIds)}}); const documentObjs: any = {}; for(let i = 0; i < collections.length; i++) { const collection = collections[i]; // generate vector from hub for the search query const vector = await this.aquilaClientService.getHubServer().compressDocument(collection.aquilaDbName, [options.query]) as number[][]; if(vector.length === 0) { throw new InternalServerError("Something went wrong"); } // search on aquiladb const { docs, dists } = await this.aquilaClientService.getDbServer().searchKDocuments(collection.aquilaDbName, vector, 10) const newDists = dists[0]; docs[0].forEach((doc: any, index: number) => { const docExists = documentObjs[doc.metadata.bookmark_id]; if(!docExists) { documentObjs[doc.metadata.bookmark_id] = { bookmarkParaId: doc.metadata.bookmark_para_id, bookmarkId: doc.metadata.bookmark_id, dist: newDists[index], paras: [{ dist: newDists[index], para: doc.metadata.para}] } } }); } const documents = Object.values(documentObjs).sort((a: any, b: any) => (b.dist - a.dist)) // sort result from aquiladb and select records within limit and offset const totalRecords = documents.length; const start = (options.page -1) * options.limit; const end = ((options.page -1) * options.limit) + options.limit; const records = documents.slice(start, end); // fetch all bookmarks const bookmarkIds = records.map((item: any) => item.bookmarkId); const bookmarks = await Bookmark.find({ where: { id: In(bookmarkIds)}}); // generate result const bookmarkData = records.map((item: any) => { const bookmark = bookmarks.find(data => data.id === item.bookmarkId) as Bookmark; return ({ id: bookmark.id, collectionId: bookmark.collectionId, url: bookmark.url, title: bookmark.title, author: bookmark.author, coverImg: bookmark.coverImg, summary: bookmark.summary, description: item.paras[0].para }); }); const output = { totalRecords, totalPages: Math.ceil(totalRecords / options.limit), currentPage: options.page, limit: options.limit, bookmarks: bookmarkData } return output; } public async getSubscriptionsByPermanentCustomerId(customerId: string, options: GetSubscriptionsByCustomerIdOptionsInputDto): Promise { const collectionSubscriptions = await CollectionSubscription.find({ where: { subscriberId: customerId }}); const collectionIds = collectionSubscriptions.map(collection => collection.collectionId); if(!options.query) { return this.getAllSubscriptionsByCustomerId(collectionIds, options); } const collections = await Collection.find({ where : { id: In(collectionIds)}}); const documentObjs: any = {}; for(let i = 0; i < collections.length; i++) { const collection = collections[i]; // generate vector from hub for the search query const vector = await this.aquilaClientService.getHubServer().compressDocument(collection.aquilaDbName, [options.query]) as number[][]; if(vector.length === 0) { throw new InternalServerError("Something went wrong"); } // search on aquiladb const { docs, dists } = await this.aquilaClientService.getDbServer().searchKDocuments(collection.aquilaDbName, vector, 10) const newDists = dists[0]; docs[0].forEach((doc: any, index: number) => { const docExists = documentObjs[doc.metadata.bookmark_id]; if(!docExists) { documentObjs[doc.metadata.bookmark_id] = { bookmarkParaId: doc.metadata.bookmark_para_id, bookmarkId: doc.metadata.bookmark_id, dist: newDists[index], paras: [{ dist: newDists[index], para: doc.metadata.para}] } } }); } const documents = Object.values(documentObjs).sort((a: any, b: any) => (b.dist - a.dist)) // sort result from aquiladb and select records within limit and offset const totalRecords = documents.length; const start = (options.page -1) * options.limit; const end = ((options.page -1) * options.limit) + options.limit; const records = documents.slice(start, end); // fetch all bookmarks const bookmarkIds = records.map((item: any) => item.bookmarkId); const bookmarks = await Bookmark.find({ where: { id: In(bookmarkIds)}}); // generate result const bookmarkData = records.map((item: any) => { const bookmark = bookmarks.find(data => data.id === item.bookmarkId) as Bookmark; return ({ id: bookmark.id, collectionId: bookmark.collectionId, url: bookmark.url, title: bookmark.title, author: bookmark.author, coverImg: bookmark.coverImg, summary: bookmark.summary, description: item.paras[0].para }); }); const output = { totalRecords, totalPages: Math.ceil(totalRecords / options.limit), currentPage: options.page, limit: options.limit, bookmarks: bookmarkData } return output; } public async getSubscriptionsByCustomerId(customerId: string, options: GetSubscriptionsByCustomerIdOptionsInputDto, accountStatus: AccountStatus): Promise { if(accountStatus == AccountStatus.TEMPORARY) { return this.getSubscriptionsByTemporaryCustomerId(customerId, options); } return this.getSubscriptionsByPermanentCustomerId(customerId, options); } public async getCustomerSubscribedCollections(customerId: string, accountStatus: AccountStatus): Promise { let collectionSubscriptions: CollectionSubscription[] | CollectionSubscriptionTemp[]; if(accountStatus === AccountStatus.TEMPORARY){ collectionSubscriptions = await CollectionSubscriptionTemp.find({ where: { subscriberId: customerId}}); }else { collectionSubscriptions = await CollectionSubscription.find({ where: { subscriberId: customerId}}); } const collectionIds = collectionSubscriptions.map(item => item.collectionId); const collections = await Collection.find({ where: { id: In(collectionIds)}, relations: {customer: true} }); return collections; } } ================================================ FILE: aquila/search/src/service/CustomerService.ts ================================================ import { Service } from "typedi"; import randomAnimalName from '../utils/randomAnimals'; import crypto from 'crypto'; import { v4 as uuidv4, parse as parseUuid } from 'uuid'; import base58 from 'bs58'; import dataSource from '../config/db'; import { CustomerTemp } from "../entity/CustomerTemp"; import { CollectionTemp } from "../entity/CollectionTemp"; import { ActivateCustomerByIdInputDataDto, CreateCustomerInputDataDto, CreateCustomerOutputDto, GetCustomerPublicInfoByIdOutputDto, GetRandomCustomerNameOutputDto, UpdateCustomerByIdInputDataDto } from "./dto/CustomerServiceDto"; import { AquilaClientService } from "../lib/AquilaClientService"; import { Customer } from "../entity/Customer"; import { NotFoundError } from "routing-controllers"; import { AccountStatus } from "./dto/AuthServiceDto"; import { BookmarkTemp } from "../entity/BookmarkTemp"; import { In } from "typeorm"; import { BookmarkPara } from "../entity/BookmarkPara"; import { BookmarkParaTemp } from "../entity/BookmarkParaTemp"; import { Collection } from "../entity/Collection"; import { Bookmark, BookmarkStatus } from "../entity/Bookmark"; import { CollectionSubscriptionTemp } from "../entity/CollectionSubscriptionTemp"; import { CollectionSubscription } from "../entity/CollectionSubscription"; @Service() export class CustomerService { public constructor(private aquilaClientService: AquilaClientService) {} public async getRandomCustomerName(): Promise { const [firstName, lastName] = randomAnimalName(); return { firstName, lastName }; } public async getCustomerById(id: string, accountStatus: AccountStatus) { if(accountStatus === AccountStatus.PERMANENT) { return await this.getPermanentCustomerById(id); } return await this.getTemporaryCustomerById(id); } private async getPermanentCustomerById(id: string): Promise { let customer: Customer | null = await Customer.findOne({ where: { id: id }}); if(!customer) { throw new NotFoundError("Customer not found"); } return customer; } private async getTemporaryCustomerById(id: string): Promise { let customer: CustomerTemp | null = await CustomerTemp.findOne({ where: { id: id }}); if(!customer) { throw new NotFoundError("Customer not found"); } return customer; } public async createCustomer(data: CreateCustomerInputDataDto): Promise { const customer = new CustomerTemp(); const collection = new CollectionTemp(); await dataSource.transaction(async transactionalEntityManager => { // create an new record in customer const {firstName, lastName} = data; const randomNumber = Math.random().toString(); const hash = crypto.createHash('sha256'); const avatar = hash.update(randomNumber).digest('hex'); const uuidV4ParsedId = new Uint8Array(parseUuid(uuidv4())); const base58String = base58.encode(uuidV4ParsedId).slice(0, 8); const timestampString = Date.now().toString().slice(0, -3); const secretKey = base58String+timestampString; const desc = 'Hi I’m using Aquila Network to help curate the web. I’ll be sharing some awesome websites with you. Don’t forget to follow me and support my work.'; customer.firstName = firstName; customer.lastName = lastName; customer.avatar = avatar; customer.secretKey = secretKey; customer.desc = desc; await transactionalEntityManager.save(customer); // create a aquilaDb const aquilaDbName = await this.aquilaClientService.createCollection(desc, secretKey); // create an new record in collection const collectionName = 'My Collection #1'; const collectionDesc = 'Hi I’m using Aquila Network to help curate the web. I’ll be sharing some awesome websites with you. Don’t forget to follow me and support my work.'; const customerId = customer.id; collection.name = collectionName; collection.desc = collectionDesc; collection.customerId = customerId; collection.aquilaDbName = aquilaDbName; await transactionalEntityManager.save(collection); }) return { customer, collection } } public async updateCustomerById(id: string, data: UpdateCustomerByIdInputDataDto) { const customer = await this.getPermanentCustomerById(id); customer.firstName = data.firstName; customer.lastName = data.lastName; customer.desc = data.desc; customer.email = data.email; customer.lightningAddress = data.lightningAddress || null; customer.save(); return customer; } public async getCustomerPublicInfoById(id: string): Promise { const customer = await this.getPermanentCustomerById(id); return { id: id, firstName: customer.firstName, lastName: customer.lastName, desc: customer.desc, customerId: customer.customerId }; } public async activateCustomerById(id: string, data: ActivateCustomerByIdInputDataDto): Promise { // get temporary custoemr const customerTemp = await this.getTemporaryCustomerById(id); // get all collections to permanent account const collectionsTemp = await CollectionTemp.find({ where: { customerId: id }}); const collectionTempIds = collectionsTemp.map(item => item.id); // get all bookmarks to permanent account const bookmarksTemp = await BookmarkTemp.find({ where: { collectionId: In(collectionTempIds) }}) const bookmarkTempIds = await bookmarksTemp.map(item => item.id); // get all bookmark paras to permanent account const bookmarkParasTemp = await BookmarkParaTemp.find({ where: { bookmarkId: In(bookmarkTempIds)}}) // get all collection subscriptions to permanent account const collectionSubTemp = await CollectionSubscriptionTemp.find({ where: { subscriberId: id}}) // create customer const customer = new Customer(); customer.id = customerTemp.id; customer.avatar = customerTemp.avatar; customer.firstName = data.firstName; customer.lastName = data.lastName; customer.email = data.email; customer.desc = data.desc; customer.secretKey = customerTemp.secretKey; customer.createdAt = customerTemp.createdAt; customer.lightningAddress = data.lightningAddress || null; // create collections const collections = collectionsTemp.map(item => { const collection = new Collection(); collection.id = item.id; collection.desc = item.desc; collection.aquilaDbName = item.aquilaDbName; collection.customerId = item.customerId; collection.name = item.name; collection.isShareable = item.isShareable; collection.indexedDocsCount = item.indexedDocsCount; collection.createdAt = item.createdAt; return collection; }); // create bookmarks const bookmarks = bookmarksTemp.map(item => { const bookmark = new Bookmark(); bookmark.id = item.id; bookmark.title = item.title; bookmark.collectionId = item.collectionId; bookmark.coverImg = item.coverImg; bookmark.author = item.author; bookmark.html = item.html; bookmark.url = item.url; bookmark.summary = item.summary; bookmark.links = item.links; bookmark.isHidden = item.isHidden; bookmark.status = item.status as unknown as BookmarkStatus; bookmark.createdAt = item.createdAt; return bookmark; }); // create bookmark paras const bookmarkParas = bookmarkParasTemp.map(item => { const bookmarkPara = new BookmarkPara(); bookmarkPara.id = item.id; bookmarkPara.content = item.content; bookmarkPara.bookmarkId = item.bookmarkId; bookmarkPara.createdAt = item.createdAt; return bookmarkPara; }); // create collection subscriptions const collectionSubs = collectionSubTemp.map(item => { const collectionSub = new CollectionSubscription(); collectionSub.id = item.id; collectionSub.subscriberId = item.subscriberId; collectionSub.collectionId = item.collectionId; collectionSub.subscribedAt = item.subscribedAt; collectionSub.createdAt = item.createdAt; return collectionSub; }); await dataSource.transaction(async transactionalEntityManager => { transactionalEntityManager.save(customer); transactionalEntityManager.save(collections); transactionalEntityManager.save(bookmarks); transactionalEntityManager.save(bookmarkParas); transactionalEntityManager.save(collectionSubs); transactionalEntityManager.remove(customerTemp); transactionalEntityManager.remove(collectionsTemp); transactionalEntityManager.remove(bookmarksTemp); transactionalEntityManager.remove(bookmarkParasTemp); transactionalEntityManager.remove(collectionSubTemp); }); return customer; } public async findCustomerByEmailId(email: string) { const customer = await Customer.findOne({where: { email }}); return customer; } } ================================================ FILE: aquila/search/src/service/dto/AuthServiceDto.ts ================================================ export enum AccountStatus { TEMPORARY= 'TEMPORARY', PERMANENT = 'PERMANENT' } export interface JwtPayload { customerId: string, accountStatus: AccountStatus } ================================================ FILE: aquila/search/src/service/dto/BookmarkServiceDto.ts ================================================ import { Bookmark } from "../../entity/Bookmark"; import { BookmarkTemp } from "../../entity/BookmarkTemp"; export interface AddBookmarkInputDto { html?: string; url: string; collectionId: string; } export interface GetBookmarksByCollectionIdOptionsInputDto { limit: number; page: number; query?: string; } export type GetAllBookmarksByCollectionIdOptionsInputDto = Omit; export interface GetFeaturedBookmarksOptionsInputDto { limit: number; page: number; } export interface BookmarkData { id: string; collectionId: string; url: string; title: string; author: string; coverImg: string; summary: string description: string; } export interface GetBookmarksByCollectionIdOutputDto { totalPages: number; totalRecords: number; currentPage: number; limit: number; bookmarks: BookmarkData[] } export interface GetFeaturedBookmarksOutputDto { totalPages: number; totalRecords: number; currentPage: number; limit: number; bookmarks: BookmarkData[] } ================================================ FILE: aquila/search/src/service/dto/CollectionServiceDto.ts ================================================ import { Collection } from "../../entity/Collection"; import { CollectionTemp } from "../../entity/CollectionTemp"; export interface GetAllCollectionsInputOptionsDto { limit: number; page: number; where?: { isShareable?: boolean; isFeatured?: boolean; } } export interface GetAllCollectionsOutputDto { totalRecords: number; totalPages: number, currentPage: number, limit: number, collections: CollectionTemp[] | Collection[] } ================================================ FILE: aquila/search/src/service/dto/CollectionSubscriptionServiceDto.ts ================================================ export interface GetSubscriptionsByCustomerIdOptionsInputDto { limit: number; page: number; query?: string; } export interface BookmarkData { id: string; collectionId: string; url: string; title: string; author: string; coverImg: string; summary: string description: string; } export interface GetSubscriptionsByCustomerIdOutputDto { totalPages: number; totalRecords: number; currentPage: number; limit: number; bookmarks: BookmarkData[] } ================================================ FILE: aquila/search/src/service/dto/CustomerServiceDto.ts ================================================ import { CollectionTemp } from "../../entity/CollectionTemp"; import { CustomerTemp } from "../../entity/CustomerTemp"; export interface CreateCustomerInputDataDto { firstName: string; lastName: string; } export interface CreateCustomerOutputDto { customer: CustomerTemp; collection: CollectionTemp; } export interface ActivateCustomerByIdInputDataDto { firstName: string; lastName: string; email: string; desc: string; lightningAddress: string; } export interface UpdateCustomerByIdInputDataDto { firstName: string; lastName: string; email: string; desc: string; lightningAddress: string; } export interface GetCustomerPublicInfoByIdOutputDto { id: string; firstName: string; lastName: string; desc: string; customerId: number; } export interface GetRandomCustomerNameOutputDto { firstName: string; lastName: string; } ================================================ FILE: aquila/search/src/utils/errors/ValidationError.ts ================================================ import { HttpError } from "routing-controllers"; export class ValidationError extends HttpError { constructor(message: string, public errors: any ) { super(400, message); this.name = "ValidationError"; Object.setPrototypeOf(this, ValidationError.prototype); } } ================================================ FILE: aquila/search/src/utils/randomAnimals.ts ================================================ const adjectives = ["furry","ferocious","dangerous","tame","agile","clever","aggressive","tiny","domestic","wild","herbivorous","carnivorous","Adorable","Aggressive","Agile","Beautiful","Bossy","Candid","Carnivorous","Clever","Cold","Colorful","Cuddly","Curious","Cute","Dangerous","Deadly","Domestic","Dominant","Energetic","Fast","Feisty","Ferocious","Fierce","Fluffy","Friendly","Furry","Fuzzy","Grumpy","Hairy","Heavy","Herbivorous","Large","Lovable","Loving","Maternal","Messy","Noisy","Nosy","Picky","Playful","Quick","Rough","Sassy","Scaly","Short","Shy","Slimy","Slow","Small","Smart","Soft","Spikey","Stinky","Strong","Stubborn","Submissive","Tall","Tame","Tenacious","Territorial","Tiny","Vicious","Warm","Wild"]; const animals = ["alligator", "anteater", "armadillo", "auroch", "axolotl", "badger", "bat", "bear", "beaver", "blobfish", "buffalo", "camel", "chameleon", "cheetah", "chipmunk", "chinchilla", "chupacabra", "cormorant", "coyote", "crow", "dingo", "dinosaur", "dog", "dolphin", "dragon", "duck", "dumbo octopus", "elephant", "ferret", "fox", "frog", "giraffe", "goose", "gopher", "grizzly", "hamster", "hedgehog", "hippo", "hyena", "jackal", "jackalope", "ibex", "ifrit", "iguana", "kangaroo", "kiwi", "koala", "kraken", "lemur", "leopard", "liger", "lion", "llama", "manatee", "mink", "monkey", "moose", "narwhal", "nyan cat", "orangutan", "otter", "panda", "penguin", "platypus", "python", "pumpkin", "quagga", "quokka", "rabbit", "raccoon", "rhino", "sheep", "shrew", "skunk", "slow loris", "squirrel", "tiger", "turtle", "unicorn", "walrus", "wolf", "wolverine", "wombat", "Aardvark","Albatross","Alligator","Alpaca","Ant","Anteater","Antelope","Ape","Armadillo","Badger","Barracuda","Bat","Bear","Beaver","Bee","Bison","Boar","Buffalo","Butterfly","Camel","Capybara","Caribou","Cassowary","Cat","Caterpillar","Cattle","Chamois","Cheetah","Chicken","Chimpanzee","Chinchilla","Chough","Clam","Cobra","Cockroach","Cod","Cormorant","Coyote","Crab","Crane","Crocodile","Crow","Curlew","Deer","Dinosaur","Dog","Dogfish","Dolphin","Dotterel","Dove","Dragonfly","Duck","Dugong","Dunlin","Eagle","Echidna","Eel","Eland","Elephant","Elk","Emu","Falcon","Ferret","Finch","Fish","Flamingo","Fly","Fox","Frog","Gaur","Gazelle","Gerbil","Giraffe","Gnat","Gnu","Goat","Goldfinch","Goldfish","Goose","Gorilla","Goshawk","Grasshopper","Grouse","Guanaco","Gull","Hamster","Hare","Hawk","Hedgehog","Heron","Herring","Hippopotamus","Hornet","Horse","Human","Hummingbird","Hyena","Ibex","Ibis","Jackal","Jaguar","Jay","Jellyfish","Kangaroo","Kingfisher","Koala","Kookabura","Kouprey","Kudu","Lapwing","Lark","Lemur","Leopard","Lion","Llama","Lobster","Locust","Loris","Louse","Lyrebird","Magpie","Mallard","Manatee","Mandrill","Mantis","Marten","Meerkat","Mink","Mole","Mongoose","Monkey","Moose","Mosquito","Mouse","Mule","Narwhal","Newt","Nightingale","Octopus","Okapi","Opossum","Oryx","Ostrich","Otter","Owl","Oyster","Panther","Parrot","Partridge","Peafowl","Pelican","Penguin","Pheasant","Pig","Pigeon","Pony","Porcupine","Porpoise","Quail","Quelea","Quetzal","Rabbit","Raccoon","Rail","Ram","Rat","Raven","Red deer","Red panda","Reindeer","Rhinoceros","Rook","Salamander","Salmon","Sand Dollar","Sandpiper","Sardine","Scorpion","Seahorse","Seal","Shark","Sheep","Shrew","Skunk","Snail","Snake","Sparrow","Spider","Spoonbill","Squid","Squirrel","Starling","Stingray","Stinkbug","Stork","Swallow","Swan","Tapir","Tarsier","Termite","Tiger","Toad","Trout","Turkey","Turtle","Viper","Vulture","Wallaby","Walrus","Wasp","Weasel","Whale","Wildcat","Wolf","Wolverine","Wombat","Woodcock","Woodpecker","Worm","Wren","Yak","Zebra"]; export default function randomAnimal(): [string, string] { return [adjectives[Math.floor(Math.random()*adjectives.length)], animals[Math.floor(Math.random()*animals.length)]]; } ================================================ FILE: aquila/search/src/utils/validate.ts ================================================ import { Request } from "express"; import { ValidationChain, validationResult } from "express-validator"; import { ValidationError } from "./errors/ValidationError"; export const validate = async (validations: ValidationChain[], req: Request) => { await Promise.all(validations.map(validation => validation.run(req))); const errors = validationResult(req); if (errors.isEmpty()) { return; } // res.status(400).json({ errors: errors.array() }); throw new ValidationError("Invalid data", errors.array()) } ================================================ FILE: aquila/search/tsconfig.json ================================================ { "compilerOptions": { /* Visit https://aka.ms/tsconfig.json to read more about this file */ /* Projects */ // "incremental": true, /* Enable incremental compilation */ // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ /* Language and Environment */ "target": "ES2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ /* Modules */ "module": "commonjs", /* Specify what module code is generated. */ "rootDir": "./src", /* Specify the root folder within your source files. */ // "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ "typeRoots": [ "src/@types", "./node_modules/@types", ], /* Specify multiple folders that act like `./node_modules/@types`. */ // "types": [], /* Specify type package names to be included without being referenced in a source file. */ // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ // "resolveJsonModule": true, /* Enable importing .json files */ // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ /* JavaScript Support */ // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */ // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ /* Emit */ // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ // "declarationMap": true, /* Create sourcemaps for d.ts files. */ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ "outDir": "./dist", /* Specify an output folder for all emitted files. */ // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ // "newLine": "crlf", /* Set the newline character for emitting files. */ // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ /* Interop Constraints */ // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */ // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ /* Type Checking */ "strict": true, /* Enable all strict type-checking options. */ // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ "strictPropertyInitialization": false, /* Check for class properties that are declared but not set in the constructor. */ // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ } } ================================================ FILE: aquila/txt_transform/Dockerfile_mercury ================================================ FROM node:12.18.1 ENV NODE_ENV=production WORKDIR /app COPY ["package.json", "package-lock.json*", "./"] RUN npm install --production COPY . . CMD [ "node", "server.js" ] ================================================ FILE: aquila/txt_transform/Dockerfile_txtpick ================================================ FROM python:3.8-slim-buster WORKDIR /app COPY requirements.txt requirements.txt RUN pip3 install -r requirements.txt COPY . . CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "--port=5008"] ================================================ FILE: aquila/txt_transform/app.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import from __future__ import division, print_function, unicode_literals import logging import re from flask import Flask, request from flask_cors import CORS app = Flask(__name__, instance_relative_config=True) # from sumy.parsers.html import HtmlParser from sumy.parsers.plaintext import PlaintextParser from sumy.nlp.tokenizers import Tokenizer from sumy.summarizers.lsa import LsaSummarizer as Summarizer # from sumy.summarizers.luhn import LuhnSummarizer as Summarizer from sumy.nlp.stemmers import Stemmer from sumy.utils import get_stop_words from bs4 import BeautifulSoup import nltk nltk.download('punkt') LANGUAGE = "english" def get_paragraphs(html_doc): cleantext = BeautifulSoup(html_doc, "lxml").text paras = [] counter = 0 pattern_ = r"\n|(?<=[a-z]\.)\s+" result = re.split(pattern_, cleantext) for para_ in result: if para_ != None: # remove citations c_pattern_ = r"\[[^\[]*?\]" para = re.sub(c_pattern_, "", para_) # remove special chars sc_pattern = r"[^A-Za-z0-9]+" para__ = re.sub(sc_pattern, "", para) if para__.strip() != "": # minimum 10 words in a sentance is needed if len(para.split()) > 10: counter += 1 paras.append(para.strip()) return paras, counter def extract_request_params (request): if not request.is_json: logging.error("Cannot parse request parameters") # request is invalid return {} # Extract JSON data data_ = request.get_json() return data_ @app.route("/process", methods=['POST']) def process (): """ Process html content """ # get parameters html = None if extract_request_params(request).get("html"): html = extract_request_params(request)["html"] if not html: # Build error response return { "success": False, "message": "Invalid parameters" }, 400 # process logic text_in, nlines = get_paragraphs(html) sentances_ret = [] parser = PlaintextParser.from_string("\n".join(text_in), Tokenizer(LANGUAGE)) stemmer = Stemmer(LANGUAGE) summarizer = Summarizer(stemmer) summarizer.stop_words = get_stop_words(LANGUAGE) SENTENCES_COUNT = int(nlines * 0.2) if SENTENCES_COUNT > 100: SENTENCES_COUNT = 100 if SENTENCES_COUNT < 1: SENTENCES_COUNT = nlines for sentence in summarizer(parser.document, SENTENCES_COUNT): sentances_ret.append(sentence._text) # Build response return { "success": True, "result": sentances_ret }, 200 # Server starter def flaskserver (): """ start server """ app.run(host='0.0.0.0', port=5008, debug=False) # Enable CORS CORS(app) if __name__ == "__main__": flaskserver() ================================================ FILE: aquila/txt_transform/package.json ================================================ { "name": "mercury", "version": "1.0.0", "description": "", "main": "index.js", "type": "module", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", "license": "ISC", "dependencies": { "@postlight/mercury-parser": "^2.2.0", "express": "^4.17.1", "node-html-parser": "^3.3.4" } } ================================================ FILE: aquila/txt_transform/requirements.txt ================================================ sumy==0.9.0 requests==2.27.1 Flask_Cors==3.0.10 Flask==2.0.3 beautifulsoup4==4.10.0 numpy==1.22.2 ================================================ FILE: aquila/txt_transform/server.js ================================================ import express from "express"; var app = express(); app.use(express.json({limit: '50mb'})); const port = 5009 import Mercury from "@postlight/mercury-parser"; // import pkg from 'node-html-parser'; // const { parse } = pkg; app.post('/process', function (req, res) { if (req.body.url && req.body.html) { Mercury.parse(req.body.url, { html: req.body.html }).then(result => { // var root = parse(result.content); res.send({ data: result }); }); } else { res.sendStatus(400); } }); app.listen(port, () => { console.log(`Example app listening at http://localhost:${port}`); }); ================================================ FILE: aquila/txt_transform/test.html ================================================
Decentralized cryptocurrency
\n' + '\n' + '\n' + '
Prevailing bitcoin logoBitcoin
Prevailing bitcoin logo
DenominationsPluralbitcoinsSymbol₿ (Unicode: U+20BF BITCOIN SIGN (HTML &#8383;))[a]Ticker symbolBTC, XBT[b]Precision10−8Subunits11000millibitcoin1100000000satoshi[2]DevelopmentOriginal author(s)Satoshi NakamotoWhite paper"Bitcoin: A Peer-to-Peer Electronic Cash System"[4]Implementation(s)Bitcoin CoreInitial release0.1.0 / 9 January 2009 (12 years ago) (2009-01-09)Latest release0.21.1 / 2 May 2021 (25 days ago) (2021-05-02)[3]Code repositoryhttps://github.com/bitcoin/bitcoinDevelopment statusActiveWebsitebitcoin.orgLedgerLedger start3 January 2009 (12 years ago) (2009-01-03)Timestamping schemeProof-of-work (partial hash inversion)Hash functionSHA-256Issuance scheduleDecentralized (block reward)
Initially ₿50 per block, halved every 210,000 blocks[7]Block reward₿6.25[c]Block time10 minutesBlock explorerMany implementationsCirculating supply₿18,660,000 (as of 20 March 2021[update])Supply limit₿21,000,000[5][d]
\n' + '
    \n' + '
  1. ^ The symbol was encoded in Unicode version 10.0 at position U+20BF BITCOIN SIGN in the Currency Symbols block in June 2017.[1]\n' + '
  2. \n' + '
  3. ^ Compatible with ISO 4217.\n' + '
  4. \n' + '
  5. ^ May 2020 to approximately 2024, halved approximately every four years\n' + '
  6. \n' + '
  7. ^ The supply will approach, but never reach, ₿21 million. Issuance will permanently halt c. 2140 at ₿20,999,999.9769.[6]:ch. 8\n' + '
  8. \n' + '
\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '

Bitcoin () is a decentralized digital currency, without a central bank or single administrator, that can be sent from user to user on the peer-to-peer bitcoin network without the need for intermediaries.[7] Transactions are verified by network nodes through cryptography and recorded in a public distributed ledger called a blockchain. The cryptocurrency was invented in 2008 by an unknown person or group of people using the name Satoshi Nakamoto.[8] The currency began use in 2009[9] when its implementation was released as open-source software.[6]:ch. 1\n' + '

Bitcoins are created as a reward for a process known as mining. They can be exchanged for other currencies, products, and services,[10] but the real-world value of the coins is extremely volatile.[11] Research produced by the University of Cambridge estimated that in 2017, there were 2.9 to 5.8 million unique users using a cryptocurrency wallet, most of them using bitcoin.[12] Users choose to participate in the digital currency for a number of reasons: ideologies such as commitment to anarchism, decentralization and libertarianism, convenience, using the currency as an investment and pseudonymity of transactions. Increased use has led to a desire among governments for regulation in order to tax, facilitate legal use in trade and for other reasons (such as investigations for money laundering and price manipulation).\n' + '

Bitcoin has been criticized for its use in illegal transactions, the large amount of electricity (and thus carbon footprint) used by mining, price volatility, and thefts from exchanges. Some economists and commentators have characterized it as a speculative bubble at various times. Bitcoin has also been used as an investment, although several regulatory agencies have issued investor alerts about bitcoin.[11][13][14]\n' + '

The word bitcoin was defined in a white paper published on 31 October 2008.[4][15] It is a compound of the words bit and coin.[16] No uniform convention for bitcoin capitalization exists; some sources use Bitcoin, capitalized, to refer to the technology and network and bitcoin, lowercase, for the unit of account.[17] The Wall Street Journal,[18] The Chronicle of Higher Education,[19] and the Oxford English Dictionary[16] advocate the use of lowercase bitcoin in all cases.\n' + '

\n' + '\n' + '\n' + '

History

\n' + '\n' + '

Creation

\n' + '

The domain name bitcoin.org was registered on 18 August 2008.[20] On 31 October 2008, a link to a paper authored by Satoshi Nakamoto titled Bitcoin: A Peer-to-Peer Electronic Cash System[4] was posted to a cryptography mailing list.[21] Nakamoto implemented the bitcoin software as open-source code and released it in January 2009.[22][23][9] Nakamoto's identity remains unknown.[8]\n' + '

On 3 January 2009, the bitcoin network was created when Nakamoto mined the starting block of the chain, known as the genesis block.[24][25] Embedded in the coinbase of this block was the text "The Times 03/Jan/2009 Chancellor on brink of second bailout for banks".[9] This note references a headline published by The Times and has been interpreted as both a timestamp and a comment on the instability caused by fractional-reserve banking.[26]:18\n' + '

The receiver of the first bitcoin transaction was cypherpunk Hal Finney, who had created the first reusable proof-of-work system (RPoW) in 2004.[27] Finney downloaded the bitcoin software on its release date, and on 12 January 2009 received ten bitcoins from Nakamoto.[28][29] Other early cypherpunk supporters were creators of bitcoin predecessors: Wei Dai, creator of b-money, and Nick Szabo, creator of bit gold.[24] In 2010, the first known commercial transaction using bitcoin occurred when programmer Laszlo Hanyecz bought two Papa John's pizzas for ₿10,000.[30]\n' + '

Blockchain analysts estimate that Nakamoto had mined about one million bitcoins[31] before disappearing in 2010 when he handed the network alert key and control of the code repository over to Gavin Andresen. Andresen later became lead developer at the Bitcoin Foundation.[32][33] Andresen then sought to decentralize control. This left opportunity for controversy to develop over the future development path of bitcoin, in contrast to the perceived authority of Nakamoto's contributions.[34][33]\n' + '

\n' + '

2011–2012

\n' + '

After early "proof-of-concept" transactions, the first major users of bitcoin were black markets, such as Silk Road. During its 30 months of existence, beginning in February 2011, Silk Road exclusively accepted bitcoins as payment, transacting 9.9 million in bitcoins, worth about $214 million.[35]:222\n' + '

In 2011, the price started at $0.30 per bitcoin, growing to $5.27 for the year. The price rose to $31.50 on 8 June. Within a month, the price fell to $11.00. The next month it fell to $7.80, and in another month to $4.77.[36]\n' + '

In 2012, bitcoin prices started at $5.27, growing to $13.30 for the year.[36] By 9 January the price had risen to $7.38, but then crashed by 49% to $3.80 over the next 16 days. The price then rose to $16.41 on 17 August, but fell by 57% to $7.10 over the next three days.[37]\n' + '

The Bitcoin Foundation was founded in September 2012 to promote bitcoin's development and uptake.[38]\n' + '

On 1 November 2011, the reference implementation Bitcoin-Qt version 0.5.0 was released. It introduced a front end that used the Qt user interface toolkit.[39] The software previously used Berkeley DB for database management. Developers switched to LevelDB in release 0.8 in order to reduce blockchain synchronization time.[citation needed] The update to this release resulted in a minor blockchain fork on 11 March 2013. The fork was resolved shortly afterwards.[citation needed] Seeding nodes through IRC was discontinued in version 0.8.2. From version 0.9.0 the software was renamed to Bitcoin Core. Transaction fees were reduced again by a factor of ten as a means to encourage microtransactions.[citation needed] Although Bitcoin Core does not use OpenSSL for the operation of the network, the software did use OpenSSL for remote procedure calls. Version 0.9.1 was released to remove the network's vulnerability to the Heartbleed bug.[citation needed]\n' + '

\n' + '

2013–2016

\n' + '

In 2013, prices started at $13.30 rising to $770 by 1 January 2014.[36]\n' + '

In March 2013 the blockchain temporarily split into two independent chains with different rules due to a bug in version 0.8 of the bitcoin software. The two blockchains operated simultaneously for six hours, each with its own version of the transaction history from the moment of the split. Normal operation was restored when the majority of the network downgraded to version 0.7 of the bitcoin software, selecting the backwards-compatible version of the blockchain. As a result, this blockchain became the longest chain and could be accepted by all participants, regardless of their bitcoin software version.[40] During the split, the Mt. Gox exchange briefly halted bitcoin deposits and the price dropped by 23% to $37[40][41] before recovering to the previous level of approximately $48 in the following hours.[42]\n' + '

The US Financial Crimes Enforcement Network (FinCEN) established regulatory guidelines for "decentralized virtual currencies" such as bitcoin, classifying American bitcoin miners who sell their generated bitcoins as Money Service Businesses (MSBs), that are subject to registration or other legal obligations.[43][44][45]\n' + '

In April, exchanges BitInstant and Mt. Gox experienced processing delays due to insufficient capacity[46] resulting in the bitcoin price dropping from $266 to $76 before returning to $160 within six hours.[47] The bitcoin price rose to $259 on 10 April, but then crashed by 83% to $45 over the next three days.[37]\n' + '

On 15 May 2013, US authorities seized accounts associated with Mt. Gox after discovering it had not registered as a money transmitter with FinCEN in the US.[48][49] On 23 June 2013, the US Drug Enforcement Administration listed ₿11.02 as a seized asset in a United States Department of Justice seizure notice pursuant to 21 U.S.C. § 881. This marked the first time a government agency had seized bitcoin.[50] The FBI seized about ₿30,000[51] in October 2013 from the dark web website Silk Road, following the arrest of Ross William Ulbricht.[52][53][54] These bitcoins were sold at blind auction by the United States Marshals Service to venture capital investor Tim Draper.[51] Bitcoin's price rose to $755 on 19 November and crashed by 50% to $378 the same day. On 30 November 2013, the price reached $1,163 before starting a long-term crash, declining by 87% to $152 in January 2015.[37]\n' + '

On 5 December 2013, the People's Bank of China prohibited Chinese financial institutions from using bitcoins.[55] After the announcement, the value of bitcoins dropped,[56] and Baidu no longer accepted bitcoins for certain services.[57] Buying real-world goods with any virtual currency had been illegal in China since at least 2009.[58]\n' + '

In 2014, prices started at $770 and fell to $314 for the year.[36] On 30 July 2014, the Wikimedia Foundation started accepting donations of bitcoin.[59]\n' + '

In 2015, prices started at $314 and rose to $434 for the year. In 2016, prices rose and climbed up to $998 by 1 January 2017.[36]\n' + '

Release 0.10 of the software was made public on 16 February 2015. It introduced a consensus library which gave programmers easy access to the rules governing consensus on the network. In version 0.11.2 developers added a new feature which allowed transactions to be made unspendable until a specific time in the future.[60] Bitcoin Core 0.12.1 was released on 15 April 2016, and enabled multiple soft forks to occur concurrently.[61] Around 100 contributors worked on Bitcoin Core 0.13.0 which was released on 23 August 2016.\n' + '

In July 2016, the CheckSequenceVerify soft fork activated.[62]\n' + '

In October 2016, Bitcoin Core's 0.13.1 release featured the "Segwit" soft fork that included a scaling improvement aiming to optimize the bitcoin blocksize.[citation needed] The patch which was originally finalised in April, and 35 developers were engaged to deploy it.[citation needed] This release featured Segregated Witness (SegWit) which aimed to place downward pressure on transaction fees as well as increase the maximum transaction capacity of the network.[63][non-primary source needed] The 0.13.1 release endured extensive testing and research leading to some delays in its release date.[citation needed] SegWit prevents various forms of transaction malleability.[64][non-primary source needed]\n' + '

\n' + '

2017–2019

\n' + '

On 15 July 2017, the controversial Segregated Witness [SegWit] software upgrade was approved ("locked-in"). Segwit was intended to support the Lightning Network as well as improve scalability.[65] SegWit was subsequently activated on the network on 24 August 2017. The bitcoin price rose almost 50% in the week following SegWit's approval.[65] On 21 July 2017, bitcoin was trading at $2,748, up 52% from 14 July 2017's $1,835.[65] Supporters of large blocks who were dissatisfied with the activation of SegWit forked the software on 1 August 2017 to create Bitcoin Cash, becoming one of many forks of bitcoin such as Bitcoin Gold.[66] \n' + '

Prices started at $998 in 2017 and rose to $13,412.44 on 1 January 2018,[36] after reaching its all-time high of $19,783.06 on 17 December 2017.[67]\n' + '

China banned trading in bitcoin, with first steps taken in September 2017, and a complete ban that started on 1 February 2018. Bitcoin prices then fell from $9,052 to $6,914 on 5 February 2018.[37] The percentage of bitcoin trading in the Chinese renminbi fell from over 90% in September 2017 to less than 1% in June 2018.[68]\n' + '

Throughout the rest of the first half of 2018, bitcoin's price fluctuated between $11,480 and $5,848. On 1 July 2018, bitcoin's price was $6,343.[69][70] The price on 1 January 2019 was $3,747, down 72% for 2018 and down 81% since the all-time high.[69][71]\n' + '

In September 2018, an anonymous party discovered and reported an invalid-block denial-of-server vulnerability to developers of Bitcoin Core, Bitcoin ABC and Bitcoin Unlimited. Further analysis by bitcoin developers showed the issue could also allow the creation of blocks violating the 21 million coin limit and CVE-2018-17144 was assigned and the issue resolved.[72][non-primary source needed]\n' + '

Bitcoin prices were negatively affected by several hacks or thefts from cryptocurrency exchanges, including thefts from Coincheck in January 2018, Bithumb in June, and Bancor in July. For the first six months of 2018, $761 million worth of cryptocurrencies was reported stolen from exchanges.[73] Bitcoin's price was affected even though other cryptocurrencies were stolen at Coinrail and Bancor as investors worried about the security of cryptocurrency exchanges.[74][75][76] In September 2019 the Intercontinental Exchange (the owner of the NYSE) began trading of bitcoin futures on its exchange called Bakkt.[77] Bakkt also announced that it would launch options on bitcoin in December 2019.[78] In December 2019, YouTube removed bitcoin and cryptocurrency videos, but later restored the content after judging they had "made the wrong call."[79]\n' + '

In February 2019, Canadian cryptocurrency exchange Quadriga Fintech Solutions failed with approximately $200 million missing.[80] By June 2019 the price had recovered to $13,000.[81]\n' + '

\n' + '

2020–present

\n' + '

According to CoinMetrics and Forbes, on 11 March 281,000 bitcoins were sold by owners who held them for only thirty days. This compared to ₿4,131 that had laid dormant for a year or more, indicating that the vast majority of the bitcoin volatility on that day was from recent buyers.[81] During the week of 11 March 2020 as a result of the COVID-19 pandemic, cryptocurrency exchange Kraken experienced an 83% increase in the number of account signups over the week of bitcoin's price collapse, a result of buyers looking to capitalize on the low price.[81] On 13 March 2020, bitcoin fell below $4000 during a broad COVID-19 pandemic related market selloff, after trading above $10,000 in February 2020.[82]\n' + '

In August 2020, MicroStrategy invested $250 million in bitcoin as a treasury reserve asset.[83] In October 2020, Square, Inc. put approximately 1% of their total assets ($50 million) in bitcoin.[84] In November 2020, PayPal announced that all users in the US could buy, hold, or sell bitcoin using PayPal.[85] On 30 November 2020, bitcoin hit a new all-time high of $19,860 topping the previous high from December 2017.[86] Alexander Vinnik, founder of BTC-e, was convicted and sentenced to 5 years in prison for money laundering in France while refusing to testify during his trial.[87] In December 2020 Massachusetts Mutual Life Insurance Company announced it has purchased $100 million in bitcoin, or roughly 0.04% of its general investment account.[88]\n' + '

On 19 January 2021, Elon Musk placed #Bitcoin in his Twitter profile tweeting "In retrospect, it was inevitable", which caused the price to briefly rise about $5000 in an hour to $37,299.[89] On 25 January 2021 Microstrategy announced it continued to buy bitcoin and as of the same date it had holdings of ₿70,784 worth $2.38 billion.[90] On 8 February 2021 Tesla's announcement that it had purchased $1.5 billion in bitcoin and planned to start accepting bitcoin as payment for vehicles pushed the bitcoin price to an all-time high of $44,141.[91] On 18 February 2021, Elon Musk said that "owning bitcoin was only a little better than holding conventional cash, but that the slight difference made it a better asset to hold".[92]\n' + '

It was announced in September 2020, that the Canton of Zug (in Switzerland) will start to accept tax payments in bitcoin from February 2021.[93][94]\n' + '

\n' + '

Design

\n' + '
Graph of the elliptic curve named secp256k1 over the algebraic number field of real numbers, \n' + ' \n' + ' \n' + ' \n' + ' R\n' + ' \n' + ' \n' + ' {\\displaystyle R}\n' + ' \n' + 'R2
\n' + '

Bitcoin is based on an elliptic curve called secp256k1 and encrypted with the ECDSA algorithm.[95][better source needed] The equation for the Bitcoin secp256k1 curve is \n' + ' \n' + ' \n' + ' \n' + ' y\n' + ' \n' + ' \n' + ' {\\displaystyle y}\n' + ' \n' + 'y2=\n' + ' \n' + ' \n' + ' \n' + ' x\n' + ' \n' + ' \n' + ' {\\displaystyle x}\n' + ' \n' + 'x3+7.[96][better source needed] Bitcoin has a proposed Bitcoin Improvement Proposal (BIP) that would add support for Schnorr signatures.[97]:101\n' + '

\n' + '

Units and divisibility

\n' + '

The unit of account of the bitcoin system is a bitcoin. Ticker symbols used to represent bitcoin are BTC[a] and XBT.[b][101]:2 Its Unicode character is ₿.[1] Small amounts of bitcoin used as alternative units are millibitcoin (mBTC), and satoshi (sat). Named in homage to bitcoin's creator, a satoshi is the smallest amount within bitcoin representing 1100000000 bitcoins, one hundred millionth of a bitcoin.[2] A millibitcoin equals 11000 bitcoins; one thousandth of a bitcoin or 100,000 satoshis.[102]\n' + '

\n' + '

Blockchain

\n' + '
Data structure of blocks in the ledger.
\n' + '
Number of bitcoin transactions per month, semilogarithmic plot[103]
\n' + '\n' + '

The bitcoin blockchain is a public ledger that records bitcoin transactions.[105] It is implemented as a chain of blocks, each block containing a hash of the previous block up to the genesis block[c] of the chain. A network of communicating nodes running bitcoin software maintains the blockchain.[35]:215–219 Transactions of the form payer X sends Y bitcoins to payee Z are broadcast to this network using readily available software applications.\n' + '

Network nodes can validate transactions, add them to their copy of the ledger, and then broadcast these ledger additions to other nodes. To achieve independent verification of the chain of ownership each network node stores its own copy of the blockchain.[106] At varying intervals of time averaging to every 10 minutes, a new group of accepted transactions, called a block, is created, added to the blockchain, and quickly published to all nodes, without requiring central oversight. This allows bitcoin software to determine when a particular bitcoin was spent, which is needed to prevent double-spending. A conventional ledger records the transfers of actual bills or promissory notes that exist apart from it, but the blockchain is the only place that bitcoins can be said to exist in the form of unspent outputs of transactions.[6]:ch. 5\n' + '

Individual blocks, public addresses and transactions within blocks can be examined using a blockchain explorer.[citation needed]\n' + '

\n' + '

Supply

\n' + '
Total bitcoins in circulation.[104]
\n' + '

The successful miner finding the new block is allowed by the rest of the network to reward themselves with newly created bitcoins and transaction fees.[107] As of 11 May 2020[update],[citation needed] the reward amounted to 6.25 newly created bitcoins per block added to the blockchain,[citation needed] plus any transaction fees from payments processed by the block. To mine half of the supply of bitcoins took four years but the remainder will take another 120 years, because of an artificial process called "bitcoin halving" according to which miners are compensated by fewer BTC as time goes on.[citation needed] To claim the reward, a special transaction called a coinbase is included with the processed payments.[6]:ch. 8 All bitcoins in existence have been created in such coinbase transactions. The bitcoin protocol specifies that the reward for adding a block will be halved every 210,000 blocks (approximately every four years). Eventually, the reward will decrease to zero, and the limit of 21 million bitcoins[d] will be reached c. 2140; the record keeping will then be rewarded solely by transaction fees.[108]\n' + '

In other words, Nakamoto set a monetary policy based on artificial scarcity at bitcoin's inception that the total number of bitcoins could never exceed 21 million. New bitcoins are created roughly every ten minutes and the rate at which they are generated drops by half about every four years until all will be in circulation.[109]\n' + '

\n' + '

Transactions

\n' + '\n' + '

Transactions are defined using a Forth-like scripting language.[6]:ch. 5 Transactions consist of one or more inputs and one or more outputs. When a user sends bitcoins, the user designates each address and the amount of bitcoin being sent to that address in an output. To prevent double spending, each input must refer to a previous unspent output in the blockchain.[110] The use of multiple inputs corresponds to the use of multiple coins in a cash transaction. Since transactions can have multiple outputs, users can send bitcoins to multiple recipients in one transaction. As in a cash transaction, the sum of inputs (coins used to pay) can exceed the intended sum of payments. In such a case, an additional output is used, returning the change back to the payer.[110] Any input satoshis not accounted for in the transaction outputs become the transaction fee.[110]\n' + '

Though transaction fees are optional, miners can choose which transactions to process and prioritize those that pay higher fees.[110] Miners may choose transactions based on the fee paid relative to their storage size, not the absolute amount of money paid as a fee. These fees are generally measured in satoshis per byte (sat/b). The size of transactions is dependent on the number of inputs used to create the transaction, and the number of outputs.[6]:ch. 8\n' + '

The blocks in the blockchain were originally limited to 32 megabytes in size. The block size limit of one megabyte was introduced by Satoshi Nakamoto in 2010. Eventually the block size limit of one megabyte created problems for transaction processing, such as increasing transaction fees and delayed processing of transactions.[111] Andreas Antonopoulos has stated Lightning Network is a potential scaling solution and referred to lightning as a second layer routing network.[6]:ch. 8\n' + '

\n' + '

Ownership

\n' + '
Simplified chain of ownership as illustrated in the bitcoin whitepaper.[4] In practice, a transaction can have more than one input and more than one output.[110]
\n' + '

In the blockchain, bitcoins are registered to bitcoin addresses. Creating a bitcoin address requires nothing more than picking a random valid private key and computing the corresponding bitcoin address. This computation can be done in a split second. But the reverse, computing the private key of a given bitcoin address, is practically unfeasible.[6]:ch. 4 Users can tell others or make public a bitcoin address without compromising its corresponding private key. Moreover, the number of valid private keys is so vast that it is extremely unlikely someone will compute a key-pair that is already in use and has funds. The vast number of valid private keys makes it unfeasible that brute force could be used to compromise a private key. To be able to spend their bitcoins, the owner must know the corresponding private key and digitally sign the transaction. The network verifies the signature using the public key; the private key is never revealed.[6]:ch. 5\n' + '

If the private key is lost, the bitcoin network will not recognize any other evidence of ownership;[35] the coins are then unusable, and effectively lost. For example, in 2013 one user claimed to have lost 7,500 bitcoins, worth $7.5 million at the time, when he accidentally discarded a hard drive containing his private key.[112] About 20% of all bitcoins are believed to be lost -they would have had a market value of about $20 billion at July 2018 prices.[113]\n' + '

To ensure the security of bitcoins, the private key must be kept secret.[6]:ch. 10 If the private key is revealed to a third party, e.g. through a data breach, the third party can use it to steal any associated bitcoins.[114] As of December 2017[update], around 980,000 bitcoins have been stolen from cryptocurrency exchanges.[115]\n' + '

Regarding ownership distribution, as of 16 March 2018, 0.5% of bitcoin wallets own 87% of all bitcoins ever mined.[116]\n' + '

\n' + '

Mining

\n' + '\n' + '
Early bitcoin miners used GPUs for mining, as they were better suited to the proof-of-work algorithm than CPUs.[117]
Later amateurs mined bitcoins with specialized FPGA and ASIC chips. The chips pictured have become obsolete due to increasing difficulty.
Today, bitcoin mining companies dedicate facilities to housing and operating large amounts of high-performance mining hardware.[118]
\n' + '
Semi-log plot of relative mining difficulty[e][104]
\n' + '

Mining is a record-keeping service done through the use of computer processing power.[f] Miners keep the blockchain consistent, complete, and unalterable by repeatedly grouping newly broadcast transactions into a block, which is then broadcast to the network and verified by recipient nodes.[105] Each block contains a SHA-256 cryptographic hash of the previous block,[105] thus linking it to the previous block and giving the blockchain its name.[6]:ch. 7[105]\n' + '

To be accepted by the rest of the network, a new block must contain a proof-of-work (PoW).[105] The system used is based on Adam Back's 1997 anti-spam scheme, Hashcash.[120][failed verification][4] The PoW requires miners to find a number called a nonce, such that when the block content is hashed along with the nonce, the result is numerically smaller than the network's difficulty target.[6]:ch. 8 This proof is easy for any node in the network to verify, but extremely time-consuming to generate, as for a secure cryptographic hash, miners must try many different nonce values (usually the sequence of tested values is the ascending natural numbers: 0, 1, 2, 3, ...[6]:ch. 8) before meeting the difficulty target.\n' + '

Every 2,016 blocks (approximately 14 days at roughly 10 min per block), the difficulty target is adjusted based on the network's recent performance, with the aim of keeping the average time between new blocks at ten minutes. In this way the system automatically adapts to the total amount of mining power on the network.[6]:ch. 8 Between 1 March 2014 and 1 March 2015, the average number of nonces miners had to try before creating a new block increased from 16.4 quintillion to 200.5 quintillion.[121]\n' + '

The proof-of-work system, alongside the chaining of blocks, makes modifications of the blockchain extremely hard, as an attacker must modify all subsequent blocks in order for the modifications of one block to be accepted.[122] As new blocks are mined all the time, the difficulty of modifying a block increases as time passes and the number of subsequent blocks (also called confirmations of the given block) increases.[105]\n' + '

Computing power is often bundled together by a Mining pool to reduce variance in miner income. Individual mining rigs often have to wait for long periods to confirm a block of transactions and receive payment. In a pool, all participating miners get paid every time a participating server solves a block. This payment depends on the amount of work an individual miner contributed to help find that block.[123]\n' + '

\n' + '

Wallets

\n' + '\n' + '
Bitcoin Core, a full client
Electrum, a lightweight client
\n' + '

The first wallet program, simply named Bitcoin, and sometimes referred to as the Satoshi client, was released in 2009 by Satoshi Nakamoto as open-source software.[9] In version 0.5 the client moved from the wxWidgets user interface toolkit to Qt, and the whole bundle was referred to as Bitcoin-Qt.[124] After the release of version 0.9, the software bundle was renamed Bitcoin Core to distinguish itself from the underlying network.[125][126] Bitcoin Core is, perhaps, the best known implementation or client. Alternative clients (forks of Bitcoin Core) exist, such as Bitcoin XT, Bitcoin Unlimited,[34] and Parity Bitcoin.[127]\n' + 'List of bitcoin forks\n' + '

A wallet stores the information necessary to transact bitcoins. While wallets are often described as a place to hold[128] or store bitcoins, due to the nature of the system, bitcoins are inseparable from the blockchain transaction ledger. A wallet is more correctly defined as something that "stores the digital credentials for your bitcoin holdings" and allows one to access (and spend) them.[6]:ch. 1, glossary Bitcoin uses public-key cryptography, in which two cryptographic keys, one public and one private, are generated.[129] At its most basic, a wallet is a collection of these keys.\n' + '

There are several modes which wallets can operate in. They have an inverse relationship with regards to trustlessness and computational requirements.\n' + '

\n' + '
  • Full clients verify transactions directly by downloading a full copy of the blockchain (over 150 GB as of January 2018[update]).[130] They are the most secure and reliable way of using the network, as trust in external parties is not required. Full clients check the validity of mined blocks, preventing them from transacting on a chain that breaks or alters network rules.[6]:ch. 1 Because of its size and complexity, downloading and verifying the entire blockchain is not suitable for all computing devices.
  • \n' + '
  • Lightweight clients consult full nodes to send and receive transactions without requiring a local copy of the entire blockchain (see simplified payment verificationSPV). This makes lightweight clients much faster to set up and allows them to be used on low-power, low-bandwidth devices such as smartphones. When using a lightweight wallet, however, the user must trust full nodes, as it can report faulty values back to the user. Lightweight clients follow the longest blockchain and do not ensure it is valid, requiring trust in full nodes.[131]
\n' + '

Third-party internet services called online wallets offer similar functionality but may be easier to use. In this case, credentials to access funds are stored with the online wallet provider rather than on the user's hardware.[132] As a result, the user must have complete trust in the online wallet provider. A malicious provider or a breach in server security may cause entrusted bitcoins to be stolen. An example of such a security breach occurred with Mt. Gox in 2011.[133]\n' + '

\n' + '
A paper wallet with a banknote-like design. Both the private key and the address are visible in text form and as 2D barcodes.
A paper wallet with the address visible for adding or checking stored funds. The part of the page containing the private key is folded over and sealed.
A brass token with a private key hidden beneath a tamper-evident security hologram. A part of the address is visible through a transparent part of the hologram.
A hardware wallet peripheral which processes bitcoin payments without exposing any credentials to the computer.
\n' + '

Physical wallets store the credentials necessary to spend bitcoins offline and can be as simple as a paper printout of the private key:[6]:ch. 10 a paper wallet or more advanced such as a hardware wallet. A paper wallet is created with a keypair generated on a computer with no internet connection; the private key is written or printed onto the paper[g] and then erased from the computer. The paper wallet can then be stored in a safe physical location for later retrieval. Bitcoins stored using a paper wallet are said to be in cold storage.[134]:39\n' + '

Cameron and Tyler Winklevoss, the founders of the Gemini Trust Co. exchange, reported that they had cut their paper wallets into pieces and stored them in envelopes distributed to safe deposit boxes across the United States.[135] Through this system, the theft of one envelope would neither allow the thief to steal any bitcoins nor deprive the rightful owners of their access to them.[136]\n' + '

Physical wallets can also take the form of metal token coins[137] with a private key accessible under a security hologram in a recess struck on the reverse side.[138]:38 The security hologram self-destructs when removed from the token, showing that the private key has been accessed.[139] Originally, these tokens were struck in brass and other base metals, but later used precious metals as bitcoin grew in value and popularity.[138]:80 Coins with stored face value as high as ₿1000 have been struck in gold.[138]:102–104 The British Museum's coin collection includes four specimens from the earliest series[138]:83 of funded bitcoin tokens; one is currently on display in the museum's money gallery.[140] In 2013, a Utahn manufacturer of these tokens was ordered by the Financial Crimes Enforcement Network (FinCEN) to register as a money services business before producing any more funded bitcoin tokens.[137][138]:80\n' + '

Another type of physical wallet called a hardware wallet keeps credentials offline while facilitating transactions.[141] The hardware wallet acts as a computer peripheral and signs transactions as requested by the user, who must press a button on the wallet to confirm that they intended to make the transaction. Hardware wallets never expose their private keys, keeping bitcoins in cold storage even when used with computers that may be compromised by malware.[134]:42–45\n' + '

\n' + '

Decentralization

\n' + '

Bitcoin is decentralized thus:[7]\n' + '

\n' + '
  • Bitcoin does not have a central authority.[7]
  • \n' + '
  • There is no central server; the bitcoin network is peer-to-peer.[9]
  • \n' + '
  • There is no central storage; the bitcoin ledger is distributed.[142]
  • \n' + '
  • The ledger is public; anybody can store it on their computer.[6]:ch. 1
  • \n' + '
  • There is no single administrator;[7] the ledger is maintained by a network of equally privileged miners.[6]:ch. 1
  • \n' + '
  • Anybody can become a miner.[6]:ch. 1
  • \n' + '
  • The additions to the ledger are maintained through competition. Until a new block is added to the ledger, it is not known which miner will create the block.[6]:ch. 1
  • \n' + '
  • The issuance of bitcoins is decentralized. They are issued as a reward for the creation of a new block.[107]
  • \n' + '
  • Anybody can create a new bitcoin address (a bitcoin counterpart of a bank account) without needing any approval.[6]:ch. 1
  • \n' + '
  • Anybody can send a transaction to the network without needing any approval; the network merely confirms that the transaction is legitimate.[143]:32
\n' + '

Conversely, researchers have pointed out at a "trend towards centralization". Although bitcoin can be sent directly from user to user, in practice intermediaries are widely used.[35]:220–222 Bitcoin miners join large mining pools to minimize the variance of their income.[35]:215, 219–222[144]:3[145] Because transactions on the network are confirmed by miners, decentralization of the network requires that no single miner or mining pool obtains 51% of the hashing power, which would allow them to double-spend coins, prevent certain transactions from being verified and prevent other miners from earning income.[146] As of 2013[update] just six mining pools controlled 75% of overall bitcoin hashing power.[146] In 2014 mining pool Ghash.io obtained 51% hashing power which raised significant controversies about the safety of the network. The pool has voluntarily capped their hashing power at 39.99% and requested other pools to act responsibly for the benefit of the whole network.[147] Around the year 2017, over 70% of the hashing power and 90% of transactions were operating from China.[148]\n' + '

According to researchers, other parts of the ecosystem are also "controlled by a small set of entities", notably the maintenance of the client software, online wallets and simplified payment verification (SPV) clients.[146]\n' + '

\n' + '

Privacy & Fungibility

\n' + '

Bitcoin is pseudonymous, meaning that funds are not tied to real-world entities but rather bitcoin addresses. Owners of bitcoin addresses are not explicitly identified, but all transactions on the blockchain are public. In addition, transactions can be linked to individuals and companies through "idioms of use" (e.g., transactions that spend coins from multiple inputs indicate that the inputs may have a common owner) and corroborating public transaction data with known information on owners of certain addresses.[149] Additionally, bitcoin exchanges, where bitcoins are traded for traditional currencies, may be required by law to collect personal information.[150] To heighten financial privacy, a new bitcoin address can be generated for each transaction.[151]\n' + '

Wallets and similar software technically handle all bitcoins as equivalent, establishing the basic level of fungibility. Researchers have pointed out that the history of each bitcoin is registered and publicly available in the blockchain ledger, and that some users may refuse to accept bitcoins coming from controversial transactions, which would harm bitcoin's fungibility.[152] For example, in 2012, Mt. Gox froze accounts of users who deposited bitcoins that were known to have just been stolen.[153]\n' + '

\n' + '

Ideology

\n' + '

Satoshi Nakamoto stated in his white paper that: "The root problem with conventional currencies is all the trust that's required to make it work. The central bank must be trusted not to debase the currency, but the history of fiat currencies is full of breaches of that trust."[154]\n' + '

\n' + '

Austrian economics roots

\n' + '

According to the European Central Bank, the decentralization of money offered by bitcoin has its theoretical roots in the Austrian school of economics, especially with Friedrich von Hayek in his book Denationalisation of Money: The Argument Refined,[155] in which Hayek advocates a complete free market in the production, distribution and management of money to end the monopoly of central banks.[156]:22\n' + '

\n' + '

Anarchism and libertarianism

\n' + '\n' + '

According to The New York Times, libertarians and anarchists were attracted to the philosophical idea behind bitcoin. Early bitcoin supporter Roger Ver said: "At first, almost everyone who got involved did so for philosophical reasons. We saw bitcoin as a great idea, as a way to separate money from the state."[154] The Economist describes bitcoin as "a techno-anarchist project to create an online version of cash, a way for people to transact without the possibility of interference from malicious governments or banks".[157] Economist Paul Krugman argues that cryptocurrencies like bitcoin are "something of a cult" based in "paranoid fantasies" of government power.[158]\n' + '

\n' + '
video iconExternal video The Declaration Of Bitcoin's Independence, BraveTheWorld, 4:38[159]

Nigel Dodd argues in The Social Life of Bitcoin that the essence of the bitcoin ideology is to remove money from social, as well as governmental, control.[160] Dodd quotes a YouTube video, with Roger Ver, Jeff Berwick, Charlie Shrem, Andreas Antonopoulos, Gavin Wood, Trace Meyer and other proponents of bitcoin reading The Declaration of Bitcoin's Independence. The declaration includes a message of crypto-anarchism with the words: "Bitcoin is inherently anti-establishment, anti-system, and anti-state. Bitcoin undermines governments and disrupts institutions because bitcoin is fundamentally humanitarian."[160][159]\n' + '

David Golumbia says that the ideas influencing bitcoin advocates emerge from right-wing extremist movements such as the Liberty Lobby and the John Birch Society and their anti-Central Bank rhetoric, or, more recently, Ron Paul and Tea Party-style libertarianism.[161] Steve Bannon, who owns a "good stake" in bitcoin, considers it to be "disruptive populism. It takes control back from central authorities. It's revolutionary."[162]\n' + '

A 2014 study of Google Trends data found correlations between bitcoin-related searches and ones related to computer programming and illegal activity, but not libertarianism or investment topics.[163]\n' + '

\n' + '

Economics

\n' + '\n' + '
Liquidity,[h] semilogarithmic plot.[104]
\n' + '

Bitcoin is a digital asset designed to work in peer-to-peer transactions as a currency.[4][164] Bitcoins have three qualities useful in a currency, according to The Economist in January 2015: they are "hard to earn, limited in supply and easy to verify."[165] Per some researchers, as of 2015[update], bitcoin functions more as a payment system than as a currency.[35]\n' + '

Economists define money as serving the following three purposes: a store of value, a medium of exchange, and a unit of account.[166] According to The Economist in 2014, bitcoin functions best as a medium of exchange.[166] However, this is debated, and a 2018 assessment by The Economist stated that cryptocurrencies met none of these three criteria.[157]\n' + 'Yale economist Robert J. Shiller writes that bitcoin has potential as a unit of account for measuring the relative value of goods, as with Chile's Unidad de Fomento, but that "Bitcoin in its present form [...] doesn't really solve any sensible economic problem".[167]\n' + '

According to research by Cambridge University, between 2.9 million and 5.8 million unique users used a cryptocurrency wallet in 2017, most of them for bitcoin. The number of users has grown significantly since 2013, when there were 300,000–1.3 million users.[12]\n' + '

\n' + '

Acceptance by merchants

\n' + '

The overwhelming majority of bitcoin transactions take place on a cryptocurrency exchange, rather than being used in transactions with merchants.[168] Delays processing payments through the blockchain of about ten minutes make bitcoin use very difficult in a retail setting. Prices are not usually quoted in units of bitcoin and many trades involve one, or sometimes two, conversions into conventional currencies.[35] Merchants that do accept bitcoin payments may use payment service providers to perform the conversions.[169]\n' + '

In 2017 and 2018 bitcoin's acceptance among major online retailers included only three of the top 500 U.S. online merchants, down from five in 2016.[168] Reasons for this decline include high transaction fees due to bitcoin's scalability issues and long transaction times.[170]\n' + '

Bloomberg reported that the largest 17 crypto merchant-processing services handled $69 million in June 2018, down from $411 million in September 2017. Bitcoin is "not actually usable" for retail transactions because of high costs and the inability to process chargebacks, according to Nicholas Weaver, a researcher quoted by Bloomberg. High price volatility and transaction fees make paying for small retail purchases with bitcoin impractical, according to economist Kim Grauer. However, bitcoin continues to be used for large-item purchases on sites such as Overstock.com, and for cross-border payments to freelancers and other vendors.[171]\n' + '

\n' + '

Financial institutions

\n' + '

Bitcoins can be bought on digital currency exchanges.\n' + '

Per researchers, "there is little sign of bitcoin use" in international remittances despite high fees charged by banks and Western Union who compete in this market.[35] The South China Morning Post, however, mentions the use of bitcoin by Hong Kong workers to transfer money home.[172]\n' + '

In 2014, the National Australia Bank closed accounts of businesses with ties to bitcoin,[173] and HSBC refused to serve a hedge fund with links to bitcoin.[174] Australian banks in general have been reported as closing down bank accounts of operators of businesses involving the currency.[175]\n' + '

On 10 December 2017, the Chicago Board Options Exchange started trading bitcoin futures,[176] followed by the Chicago Mercantile Exchange, which started trading bitcoin futures on 17 December 2017.[177]\n' + '

In September 2019 the Central Bank of Venezuela, at the request of PDVSA, ran tests to determine if bitcoin and ether could be held in central bank's reserves. The request was motivated by oil company's goal to pay its suppliers.[178]\n' + '

\n' + '

As an investment

\n' + '

The Winklevoss twins have purchased bitcoin. In 2013, The Washington Post reported a claim that they owned 1% of all the bitcoins in existence at the time.[179]\n' + '

Other methods of investment are bitcoin funds. The first regulated bitcoin fund was established in Jersey in July 2014 and approved by the Jersey Financial Services Commission.[180]\n' + '

Forbes named bitcoin the best investment of 2013.[181] In 2014, Bloomberg named bitcoin one of its worst investments of the year.[182] In 2015, bitcoin topped Bloomberg's currency tables.[183]\n' + '

According to bitinfocharts.com, in 2017 there are 9,272 bitcoin wallets with more than $1 million worth of bitcoins.[184] The exact number of bitcoin millionaires is uncertain as a single person can have more than one bitcoin wallet.\n' + '

In August 2020, MicroStrategy invested in Bitcoin.[185][186]\n' + '

In May 2021, the Bitcoin's market share on exchanges dropped from 70% to 45% as investors pursued altcoins.[187]\n' + '

\n' + '

Venture capital

\n' + '

Peter Thiel's Founders Fund invested US$3 million in BitPay.[188] In 2012, an incubator for bitcoin-focused start-ups was founded by Adam Draper, with financing help from his father, venture capitalist Tim Draper, one of the largest bitcoin holders after winning an auction of 30,000 bitcoins,[189] at the time called "mystery buyer".[190] The company's goal is to fund 100 bitcoin businesses within 2–3 years with $10,000 to $20,000 for a 6% stake.[189] Investors also invest in bitcoin mining.[191] According to a 2015 study by Paolo Tasca, bitcoin startups raised almost $1 billion in three years (Q1 2012 – Q1 2015).[192]\n' + '

\n' + '

Price and volatility

\n' + '
Price in US$, semilogarithmic plot.[104]
Annual volatility[103]
\n' + '

The price of bitcoins has gone through cycles of appreciation and depreciation referred to by some as bubbles and busts.[193] In 2011, the value of one bitcoin rapidly rose from about US$0.30 to US$32 before returning to US$2.[194] In the latter half of 2012 and during the 2012–13 Cypriot financial crisis, the bitcoin price began to rise,[195] reaching a high of US$266 on 10 April 2013, before crashing to around US$50. On 29 November 2013, the cost of one bitcoin rose to a peak of US$1,242.[196] In 2014, the price fell sharply, and as of April remained depressed at little more than half 2013 prices. As of August 2014[update] it was under US$600.[197]\n' + '

According to Mark T. Williams, as of 30 September 2014[update], bitcoin has volatility seven times greater than gold, eight times greater than the S&P 500, and 18 times greater than the US dollar.[198] Hodl is a meme created in reference to holding (as opposed to selling) during periods of volatility. Unusual for an asset, bitcoin weekend trading during December 2020 was higher than for weekdays.[199] Hedge funds (using high leverage and derivates)[200] have attempted to use the volatility to profit from downward price movements. At the end of January 2021, such positions were over $1 billion, their highest of all time.[201][202]\n' + 'As of 8 February 2021[update], the closing price of bitcoin equals US$44,797.[203]\n' + '

\n' + '

Legal status, tax and regulation

\n' + '\n' + '

Because of bitcoin's decentralized nature and its trading on online exchanges located in many countries, regulation of bitcoin has been difficult. However, the use of bitcoin can be criminalized, and shutting down exchanges and the peer-to-peer economy in a given country would constitute a de facto ban.[204] The legal status of bitcoin varies substantially from country to country and is still undefined or changing in many of them. Regulations and bans that apply to bitcoin probably extend to similar cryptocurrency systems.[192]\n' + '

According to the Library of Congress, an "absolute ban" on trading or using cryptocurrencies applies in nine countries: Algeria, Bolivia, Egypt, Iraq, Morocco, Nepal, Pakistan, Vietnam, and the United Arab Emirates. An "implicit ban" applies in another 15 countries, which include Bahrain, Bangladesh, China, Colombia, the Dominican Republic, Indonesia, Kuwait, Lesotho, Lithuania, Macau, Oman, Qatar, Saudi Arabia and Taiwan.[205]\n' + '

In October 2020, the Islamic Republic News Agency announced pending regulations that would require bitcoin miners in Iran to sell bitcoin to the Central Bank of Iran, and the central bank would use it for imports.[206] Iran, as of October 2020, had issued over 1,000 bitcoin mining licenses.[206] The Iranian government initially took a stance against cryptocurrency, but later changed it after seeing that digital currency could be used to circumvent sanctions.[207] The US Office of Foreign Assets Control listed two Iranians and their bitcoin addresses as part of its Specially Designated Nationals and Blocked Persons List for their role in the 2018 Atlanta cyberattack whose ransom was paid in bitcoin.[208]\n' + '

\n' + '

Regulatory warnings

\n' + '

The U.S. Commodity Futures Trading Commission has issued four "Customer Advisories" for bitcoin and related investments.[13] A July 2018 warning emphasized that trading in any cryptocurrency is often speculative, and there is a risk of theft from hacking, and fraud.[209] In May 2014 the U.S. Securities and Exchange Commission warned that investments involving bitcoin might have high rates of fraud, and that investors might be solicited on social media sites.[210] An earlier "Investor Alert" warned about the use of bitcoin in Ponzi schemes.[211]\n' + '

The European Banking Authority issued a warning in 2013 focusing on the lack of regulation of bitcoin, the chance that exchanges would be hacked, the volatility of bitcoin's price, and general fraud.[212] FINRA and the North American Securities Administrators Association have both issued investor alerts about bitcoin.[213][214]\n' + '

\n' + '

Price manipulation investigation

\n' + '

An official investigation into bitcoin traders was reported in May 2018.[215] The U.S. Justice Department launched an investigation into possible price manipulation, including the techniques of spoofing and wash trades.[216][217][218]\n' + '

The U.S. federal investigation was prompted by concerns of possible manipulation during futures settlement dates. The final settlement price of CME bitcoin futures is determined by prices on four exchanges, Bitstamp, Coinbase, itBit and Kraken. Following the first delivery date in January 2018, the CME requested extensive detailed trading information but several of the exchanges refused to provide it and later provided only limited data. The Commodity Futures Trading Commission then subpoenaed the data from the exchanges.[219][220]\n' + '

State and provincial securities regulators, coordinated through the North American Securities Administrators Association, are investigating "bitcoin scams" and ICOs in 40 jurisdictions.[221]\n' + '

Academic research published in the Journal of Monetary Economics concluded that price manipulation occurred during the Mt Gox bitcoin theft and that the market remains vulnerable to manipulation.[222] The history of hacks, fraud and theft involving bitcoin dates back to at least 2011.[223]\n' + '

Research by John M. Griffin and Amin Shams in 2018 suggests that trading associated with increases in the amount of the Tether cryptocurrency and associated trading at the Bitfinex exchange account for about half of the price increase in bitcoin in late 2017.[224][225]\n' + '

J.L. van der Velde, CEO of both Bitfinex and Tether, denied the claims of price manipulation: "Bitfinex nor Tether is, or has ever, engaged in any sort of market or price manipulation. Tether issuances cannot be used to prop up the price of bitcoin or any other coin/token on Bitfinex."[226]\n' + '

\n' + '

Analysis

\n' + '
video iconExternal video Cryptocurrencies: looking beyond the hype, Hyun Song Shin, Bank for International Settlements, 2:48[227]
\n' + '

The Bank for International Settlements summarized several criticisms of bitcoin in Chapter V of their 2018 annual report. The criticisms include the lack of stability in bitcoin's price, the high energy consumption, high and variable transactions costs, the poor security and fraud at cryptocurrency exchanges, vulnerability to debasement (from forking), and the influence of miners.[227][228][229]\n' + '

François R. Velde, Senior Economist at the Chicago Fed, described bitcoin as "an elegant solution to the problem of creating a digital currency".[230] David Andolfatto, Vice President at the Federal Reserve Bank of St. Louis, stated that bitcoin is a threat to the establishment, which he argues is a good thing for the Federal Reserve System and other central banks, because it prompts these institutions to operate sound policies.[119]:33[231][232]\n' + '

\n' + '

Economic concerns

\n' + '\n' + '
Bitcoin price bubbles in 2011, 2013 and 2017
\n' + '

Bitcoin, along with other cryptocurrencies, has been described as an economic bubble by at least eight Nobel Memorial Prize in Economic Sciences laureates at various times, including Robert Shiller on 1 March 2014,[167] Joseph Stiglitz on 29 November 2017,[233] and Richard Thaler on 21 December 2017.[234][235] On 29 January 2018, a noted Keynesian economist Paul Krugman has described bitcoin as "a bubble wrapped in techno-mysticism inside a cocoon of libertarian ideology",[158] on 2 February 2018, professor Nouriel Roubini of New York University has called bitcoin the "mother of all bubbles",[236] and on 27 April 2018, a University of Chicago economist James Heckman has compared it to the 17th-century tulip mania.[235]\n' + '

Journalists, economists, investors, and the central bank of Estonia have voiced concerns that bitcoin is a Ponzi scheme.[237][238][239][240] In April 2013, Eric Posner, a law professor at the University of Chicago, stated that "a real Ponzi scheme takes fraud; bitcoin, by contrast, seems more like a collective delusion."[241] A July 2014 report by the World Bank concluded that bitcoin was not a deliberate Ponzi scheme.[242]:7 In June 2014, the Swiss Federal Council examined concerns that bitcoin might be a pyramid scheme, and concluded that "since in the case of bitcoin the typical promises of profits are lacking, it cannot be assumed that bitcoin is a pyramid scheme."[243]:21\n' + '

\n' + '

Energy consumption and carbon footprint

\n' + '
Bitcoin electricity consumption
Electricity consumption of the bitcoin network since 2016 (annualized) and comparison with the electricity consumption of various countries in 2019. The upper and lower bounds (grey traces) are based on worst-case and best-case scenario assumptions, respectively. The red trace indicates an intermediate best-guess estimate. (data sources: Cambridge Bitcoin Electricity Consumption Index, US Energy Information Administration; for details, see methodology)
\n' + '

Bitcoin has been criticized for the amount of electricity consumed by mining.\n' + '

As of 2015[update], estimated combined electricity consumption attributed to mining was 166.7 megawatts and by 2017, was estimated to be between one and four gigawatts of electricity.[244][165] In 2018, bitcoin was estimated by to use 2.55 to 3.572 GW, or around 6% of the total power consumed by the global banking sector.[245][246][247] In July 2019 BBC reported bitcoin consumes about 7 gigawatts, 0.2% of the global total, or equivalent to that of Switzerland.[248] A 2021 estimate from the University of Cambridge suggests bitcoin consumes more than 178 (TWh) annually, ranking it in the top 30 energy consumers if it were a country.[249]\n' + '

Bitcoin is mined in places like Iceland where geothermal energy is cheap and cooling Arctic air is free.[250] Bitcoin miners are known to use hydroelectric power in Tibet, Quebec, Washington (state), and Austria to reduce electricity costs.[245][251] Miners are attracted to suppliers such as Hydro Quebec that have energy surpluses.[252] \n' + '

According to a University of Cambridge study, much of bitcoin mining is done in China, where electricity is subsidized by the government.[253][254] A significant part of Bitcoin mining is powered by cheap electricity in Xinjiang, which mostly comes from coal power.[255][256] In April 2021 a coal mine explosion in the province coincided with a 35% drop in hashing power and a flash crash in price.[257][255] In other provinces, such as Hunan and Sichuan, mining farms use more hydropower, however these account for at most 4% of hash power. According to Alex de Vries, renewable energy is not a good match for Bitcoin mining as 24/7 operations are best for ROI on mining devices.[257] In 2021, a US company purchased the Greenidge coal power plant and converted it to burn natural gas for the sole purpose of mining bitcoin, which has proven to be highly profitable, in spite of protests of local residents against air pollution and thermal pollution.[258]\n' + '

Concerns about bitcoin's environmental impact relate bitcoin's energy consumption to carbon emissions.[259][260] The difficulty of translating the energy consumption into carbon emissions lies in the decentralized nature of bitcoin impeding the localization of miners to examine the electricity mix used. The results of recent studies analyzing bitcoin's carbon footprint vary.[261][262][263][264] A study published in Nature Climate Change in 2018 claims that bitcoin "could alone produce enough CO
2
emissions to push warming above 2 °C within less than three decades."[263] However, other researchers criticized this analysis, arguing the underlying scenarios were inadequate, leading to overestimations.[265][266][267] According to studies published in Joule and American Chemical Society in 2019, bitcoin's annual energy consumption results in annual carbon emission ranging from 17[247] to 22.9 MtCO
2
which is comparable to the level of emissions of countries as Jordan and Sri Lanka or Kansas City.[264] George Kamiya, writing for the International Energy Agency, says that "predictions about bitcoin consuming the entire world's electricity" are sensational, but that the area "requires careful monitoring and rigorous analysis".[268]\n' + '

\n' + '

Use in illegal transactions

\n' + '\n' + '

Bitcoin held at exchanges are vulnerable to theft through phishing, scamming, and hacking. As of December 2017[update], around 980,000 bitcoins have been stolen from cryptocurrency exchanges.[115]\n' + '

The use of bitcoin by criminals has attracted the attention of financial regulators, legislative bodies, law enforcement, and the media.[269] Bitcoin gained early notoriety for its use on the Silk Road. The U.S. Senate held a hearing on virtual currencies in November 2013.[270] The U.S. government claimed that bitcoin was used to facilitate payments related to Russian interference in the 2016 United States elections.[271] However, a 2021 study led by former CIA director Michael Morell showed that broad generalizations about the use of bitcoin in illicit finance are significantly overstated and that blockchain analysis is an effective crime fighting and intelligence gathering tool.[272]\n' + '

Several news outlets have asserted that the popularity of bitcoins hinges on the ability to use them to purchase illegal goods.[164][273] Nobel-prize winning economist Joseph Stiglitz says that bitcoin's anonymity encourages money laundering and other crimes.[274][275]\n' + '

In 2014, researchers at the University of Kentucky found "robust evidence that computer programming enthusiasts and illegal activity drive interest in bitcoin, and find limited or no support for political and investment motives".[163] Australian researchers have estimated that 25% of all bitcoin users and 44% of all bitcoin transactions are associated with illegal activity as of April 2017[update]. There were an estimated 24 million bitcoin users primarily using bitcoin for illegal activity. They held $8 billion worth of bitcoin, and made 36 million transactions valued at $72 billion.[276][277]\n' + '

\n' + '

Software implementation

\n' + '
Bitcoin-core-v0.10.0.png
Bitcoin Core
The start screen under Fedora
Original author(s)Satoshi NakamotoInitial release2009Stable release0.20.1 (2 August 2020; 9 months ago (2020-08-02)) [±]\n' + 'Repositorygithub.com/bitcoin/bitcoinWritten inC++Operating systemLinux, Windows, macOSTypeCryptocurrencyLicenseMIT LicenseWebsitebitcoincore.org
\n' + '

Bitcoin Core is free and open-source software that serves as a bitcoin node (the set of which form the bitcoin network) and provides a bitcoin wallet which fully verifies payments. It is considered to be bitcoin's reference implementation.[278] Initially, the software was published by Satoshi Nakamoto under the name "Bitcoin", and later renamed to "Bitcoin Core" to distinguish it from the network.[279] It is also known as the Satoshi client.[280]\n' + '

The MIT Digital Currency Initiative funds some of the development of Bitcoin Core.[281] The project also maintains the cryptography library libsecp256k1.[282]\n' + '

Bitcoin Core includes a transaction verification engine and connects to the bitcoin network as a full node.[280] Moreover, a cryptocurrency wallet, which can be used to transfer funds, is included by default.[282] The wallet allows for the sending and receiving of bitcoins. It does not facilitate the buying or selling of bitcoin. It allows users to generate QR codes to receive payment.\n' + '

The software validates the entire blockchain, which includes all bitcoin transactions ever. This distributed ledger which has reached more than 235 gigabytes in size as of Jan 2019, must be downloaded or synchronized before full participation of the client may occur.[280] Although the complete blockchain is not needed all at once since it is possible to run in pruning mode. A command line-based daemon with a JSON-RPC interface, bitcoind, is bundled with Bitcoin Core. It also provides access to testnet, a global testing environment that imitates the bitcoin main network using an alternative blockchain where valueless "test bitcoins" are used. Regtest or Regression Test Mode creates a private blockchain which is used as a local testing environment.[283] Finally, bitcoin-cli, a simple program which allows users to send RPC commands to bitcoind, is also included.\n' + '

Checkpoints which have been hard coded into the client are used only to prevent Denial of Service attacks against nodes which are initially syncing the chain. For this reason the checkpoints included are only as of several years ago.[284][285][failed verification] A one megabyte block size limit was added in 2010 by Satoshi Nakamoto. This limited the maximum network capacity to about three transactions per second.[286] Since then, network capacity has been improved incrementally both through block size increases and improved wallet behavior. A network alert system was included by Satoshi Nakamoto as a way of informing users of important news regarding bitcoin.[287] In November 2016 it was retired. It had become obsolete as news on bitcoin is now widely disseminated.\n' + '

Bitcoin Core includes a scripting language inspired by Forth that can define transactions and specify parameters.[288] ScriptPubKey is used to "lock" transactions based on a set of future conditions. scriptSig is used to meet these conditions or "unlock" a transaction. Operations on the data are performed by various OP_Codes. Two stacks are used - main and alt. Looping is forbidden.\n' + '

Bitcoin Core uses OpenTimestamps to timestamp merge commits.[289]\n' + '

The original creator of the bitcoin client has described their approach to the software's authorship as it being written first to prove to themselves that the concept of purely peer-to-peer electronic cash was valid and that a paper with solutions could be written. The lead developer is Wladimir J. van der Laan, who took over the role on 8 April 2014.[290] Gavin Andresen was the former lead maintainer for the software client. Andresen left the role of lead developer for bitcoin to work on the strategic development of its technology.[290] Bitcoin Core in 2015 was central to a dispute with Bitcoin XT, a competing client that sought to increase the blocksize.[291] Over a dozen different companies and industry groups fund the development of Bitcoin Core.\n' + '

\n' + '

In popular culture

\n' + '

Term "HODL"

\n' + '

Hodl (/ˈhɒdəl/ HOD-əl; often written HODL) is slang in the cryptocurrency community for holding a cryptocurrency rather than selling it.[292] A person who does this is known as a Hodler. It originated in a December 2013 post on the Bitcoin Forum message board by an apparently inebriated user who posted with a typo in the subject, "I AM HODLING."[293] It is often humorously suggested to be a backronym to "hold on for dear life".[294] In 2017, Quartz listed it as one of the essential slang terms in Bitcoin culture, and described it as a stance, "to stay invested in bitcoin and not to capitulate in the face of plunging prices."[295] TheStreet.com referred to it as the "favorite mantra" of Bitcoin holders.[296] Bloomberg News referred to it as a "mantra" for holders during market routs.[297]\n' + '

\n' + '

Literature

\n' + '

In Charles Stross' 2013 science fiction novel, Neptune's Brood, the universal interstellar payment system is known as "bitcoin" and operates using cryptography.[298] Stross later blogged that the reference was intentional, saying "I wrote Neptune's Brood in 2011. Bitcoin was obscure back then, and I figured had just enough name recognition to be a useful term for an interstellar currency: it'd clue people in that it was a networked digital currency."[299]\n' + '

\n' + '

Film

\n' + '

The 2014 documentary The Rise and Rise of Bitcoin portrays the diversity of motives behind the use of bitcoin by interviewing people who use it. These include a computer programmer and a drug dealer.[300] The 2016 documentary Banking on Bitcoin is an introduction to the beginnings of bitcoin and the ideas behind cryptocurrency today.[301]\n' + '

\n' + '

Academia

\n' + '

In September 2015, the establishment of the peer-reviewed academic journal Ledger (ISSN 2379-5980) was announced. It covers studies of cryptocurrencies and related technologies, and is published by the University of Pittsburgh.[302] The journal encourages authors to digitally sign a file hash of submitted papers, which will then be timestamped into the bitcoin blockchain. Authors are also asked to include a personal bitcoin address in the first page of their papers.[303][304]\n' + '

\n' + '

See also

\n' + '\n' + '\n' + '

Notes

\n' + '
\n' + '
    \n' + '
  1. ^ As of 2014[update], BTC is a commonly used code. It does not conform to ISO 4217 as BT is the country code of Bhutan, and ISO 4217 requires the first letter used in global commodities to be 'X'.\n' + '
  2. \n' + '
  3. ^ As of 2014[update], XBT, a code that conforms to ISO 4217 though is not officially part of it, is used by Bloomberg L.P.,[98] CNNMoney,[99] and xe.com.[100]\n' + '
  4. \n' + '
  5. ^ The genesis block is the block number 0. The timestamp of the block is 2009-01-03 18:15:05. This block is unlike all other blocks in that it does not have a previous block to reference.\n' + '
  6. \n' + '
  7. ^ The exact number is 20,999,999.9769 bitcoins.[6]:ch. 8\n' + '
  8. \n' + '
  9. ^ Relative mining difficulty is defined as the ratio of the difficulty target on 9 January 2009 to the current difficulty target.\n' + '
  10. \n' + '
  11. ^ It is misleading to think that there is an analogy between gold mining and bitcoin mining. The fact is that gold miners are rewarded for producing gold, while bitcoin miners are not rewarded for producing bitcoins; they are rewarded for their record-keeping services.[119]\n' + '
  12. \n' + '
  13. ^ The private key can be printed as a series of letters and numbers, a seed phrase, or a 2D barcode. Usually, the public key or bitcoin address is also printed, so that a holder of a paper wallet can check or add funds without exposing the private key to a device.\n' + '
  14. \n' + '
  15. ^ Liquidity is estimated by a 365-day running sum of transaction outputs in USD.\n' + '
  16. \n' + '
\n' + '

References

\n' + '
\n' + '
    \n' + '
  1. ^ a b "Unicode 10.0.0". Unicode Consortium. 20 June 2017. Archived from the original on 20 June 2017. Retrieved 20 June 2017.\n' + '
  2. \n' + '
  3. ^ a b Mick, Jason (12 June 2011). "Cracking the Bitcoin: Digging Into a $131M USD Virtual Currency". Daily Tech. Archived from the original on 20 January 2013. Retrieved 30 September 2012.\n' + '
  4. \n' + '
  5. ^ "Releases - bitcoin/bitcoin". Retrieved 9 May 2021 – via GitHub.\n' + '
  6. \n' + '
  7. ^ a b c d e f Nakamoto, Satoshi (31 October 2008). "Bitcoin: A Peer-to-Peer Electronic Cash System" (PDF). bitcoin.org. Archived (PDF) from the original on 20 March 2014. Retrieved 28 April 2014.\n' + '
  8. \n' + '
  9. ^ Nakamoto; et al. (1 April 2016). "Bitcoin source code - amount constraints". Archived from the original on 1 July 2018.\n' + '
  10. \n' + '
  11. ^ a b c d e f g h i j k l m n o p q r s t u v w Antonopoulos, Andreas M. (April 2014). Mastering Bitcoin: Unlocking Digital Crypto-Currencies. O'Reilly Media. ISBN 978-1-4493-7404-4.\n' + '
  12. \n' + '
  13. ^ a b c d e "Statement of Jennifer Shasky Calvery, Director Financial Crimes Enforcement Network United States Department of the Treasury Before the United States Senate Committee on Banking, Housing, and Urban Affairs Subcommittee on National Security and International Trade and Finance Subcommittee on Economic Policy" (PDF). fincen.gov. Financial Crimes Enforcement Network. 19 November 2013. Archived (PDF) from the original on 9 October 2016. Retrieved 1 June 2014.\n' + '
  14. \n' + '
  15. ^ a b S., L. (2 November 2015). "Who is Satoshi Nakamoto?". The Economist. The Economist Newspaper Limited. Archived from the original on 21 August 2016. Retrieved 23 September 2016.\n' + '
  16. \n' + '
  17. ^ a b c d e Davis, Joshua (10 October 2011). "The Crypto-Currency: Bitcoin and its mysterious inventor". The New Yorker. Archived from the original on 1 November 2014. Retrieved 31 October 2014.\n' + '
  18. \n' + '
  19. ^ "What is Bitcoin?". CNN Money. Archived from the original on 31 October 2015. Retrieved 16 November 2015.\n' + '
  20. \n' + '
  21. ^ a b GmbH, finanzen net. "Bitcoin's limited real-world use and extreme volatility show its recent surge is still a speculative bubble, UBS Global Wealth Management says". markets.businessinsider.com. Retrieved 28 March 2021.\n' + '
  22. \n' + '
  23. ^ a b Hileman, Garrick; Rauchs, Michel. "Global Cryptocurrency Benchmarking Study" (PDF). Cambridge University. Archived (PDF) from the original on 10 April 2017. Retrieved 14 April 2017.\n' + '
  24. \n' + '
  25. ^ a b "Bitcoin". U.S. Commodity Futures Trading Commission. Archived from the original on 1 July 2018. Retrieved 17 July 2018.\n' + '
  26. \n' + '
  27. ^ "SEC.gov | INVESTOR ALERT: BITCOIN AND OTHER VIRTUAL CURRENCY-RELATED INVESTMENTS". www.sec.gov. Archived from the original on 3 June 2019. Retrieved 1 June 2019.\n' + '
  28. \n' + '
  29. ^ Vigna, Paul; Casey, Michael J. (January 2015). The Age of Cryptocurrency: How Bitcoin and Digital Money Are Challenging the Global Economic Order (1 ed.). New York: St. Martin's Press. ISBN 978-1-250-06563-6.\n' + '
  30. \n' + '
  31. ^ a b "bitcoin". OxfordDictionaries.com. Archived from the original on 2 January 2015. Retrieved 28 December 2014.\n' + '
  32. \n' + '
  33. ^ Bustillos, Maria (2 April 2013). "The Bitcoin Boom". The New Yorker. Condé Nast. Archived from the original on 27 July 2014. Retrieved 22 December 2013. Standards vary, but there seems to be a consensus forming around Bitcoin, capitalized, for the system, the software, and the network it runs on, and bitcoin, lowercase, for the currency itself.\n' + '
  34. \n' + '
  35. ^ Vigna, Paul (3 March 2014). "BitBeat: Is It Bitcoin, or bitcoin? The Orthography of the Cryptography". WSJ. Archived from the original on 19 April 2014. Retrieved 21 April 2014.\n' + '
  36. \n' + '
  37. ^ Metcalf, Allan (14 April 2014). "The latest style". Lingua Franca blog. The Chronicle of Higher Education (chronicle.com). Archived from the original on 16 April 2014. Retrieved 19 April 2014.\n' + '
  38. \n' + '
  39. ^ Bernard, Zoë (2 December 2017). "Everything you need to know about Bitcoin, its mysterious origins, and the many alleged identities of its creator". Business Insider. Archived from the original on 15 June 2018. Retrieved 15 June 2018.\n' + '
  40. \n' + '
  41. ^ Finley, Klint (31 October 2018). "After 10 Years, Bitcoin Has Changed Everything—And Nothing". Wired. Archived from the original on 5 November 2018. Retrieved 9 November 2018.\n' + '
  42. \n' + '
  43. ^ Nakamoto, Satoshi (3 January 2009). "Bitcoin". Archived from the original on 21 July 2017.\n' + '
  44. \n' + '
  45. ^ Nakamoto, Satoshi (9 January 2009). "Bitcoin v0.1 released". Archived from the original on 26 March 2014.\n' + '
  46. \n' + '
  47. ^ a b Wallace, Benjamin (23 November 2011). "The Rise and Fall of Bitcoin". Wired. Archived from the original on 31 October 2013. Retrieved 13 October 2012.\n' + '
  48. \n' + '
  49. ^ "Block 0 – Bitcoin Block Explorer". Archived from the original on 15 October 2013.\n' + '
  50. \n' + '
  51. ^ Pagliery, Jose (2014). Bitcoin: And the Future of Money. Triumph Books. ISBN 9781629370361. Archived from the original on 21 January 2018. Retrieved 20 January 2018.\n' + '
  52. \n' + '
  53. ^ "Here's The Problem with the New Theory That A Japanese Math Professor Is The Inventor of Bitcoin". San Francisco Chronicle. 19 May 2013. Archived from the original on 4 January 2015. Retrieved 24 February 2015.\n' + '
  54. \n' + '
  55. ^ Peterson, Andrea (3 January 2014). "Hal Finney received the first Bitcoin transaction. Here's how he describes it". The Washington Post. Archived from the original on 27 February 2015.\n' + '
  56. \n' + '
  57. ^ Popper, Nathaniel (30 August 2014). "Hal Finney, Cryptographer and Bitcoin Pioneer, Dies at 58". NYTimes. Archived from the original on 3 September 2014. Retrieved 2 September 2014.\n' + '
  58. \n' + '
  59. ^ Kharpal, Arjun (18 June 2018). "Everything you need to know about the blockchain". CNBC. Archived from the original on 13 September 2018. Retrieved 13 September 2018.\n' + '
  60. \n' + '
  61. ^ McMillan, Robert. "Who Owns the World's Biggest Bitcoin Wallet? The FBI". Wired. Condé Nast. Archived from the original on 21 October 2016. Retrieved 7 October 2016.\n' + '
  62. \n' + '
  63. ^ Simonite, Tom. "Meet Gavin Andresen, the most powerful person in the world of Bitcoin". MIT Technology Review. Retrieved 6 December 2017.\n' + '
  64. \n' + '
  65. ^ a b Odell, Matt (21 September 2015). "A Solution To Bitcoin's Governance Problem". TechCrunch. Archived from the original on 26 January 2016. Retrieved 24 January 2016.\n' + '
  66. \n' + '
  67. ^ a b Vigna, Paul (17 January 2016). "Is Bitcoin Breaking Up?". The Wall Street Journal. Archived from the original on 20 August 2016. Retrieved 8 November 2016.\n' + '
  68. \n' + '
  69. ^ a b c d e f g h Rainer Böhme; Nicolas Christin; Benjamin Edelman; Tyler Moore (2015). "Bitcoin: Economics, Technology, and Governance". Journal of Economic Perspectives. 29 (2): 213–238. doi:10.1257/jep.29.2.213.\n' + '
  70. \n' + '
  71. ^ a b c d e f "Bitcoin Historical Prices". OfficialData.org. Archived from the original on 4 July 2018. Retrieved 3 July 2018.\n' + '
  72. \n' + '
  73. ^ a b c d French, Sally (9 February 2017). "Here's proof that this bitcoin crash is far from the worst the cryptocurrency has seen". Market Watch. Archived from the original on 3 July 2018. Retrieved 3 July 2018.\n' + '
  74. \n' + '
  75. ^ Bustillos, Maria (1 April 2013). "The Bitcoin Boom". The New Yorker. Archived from the original on 2 July 2018. Retrieved 30 July 2018.\n' + '
  76. \n' + '
  77. ^ "Bitcoin-Qt version 0.5.0 released". Bitcoin Project. 1 November 2011. Retrieved 13 November 2016.\n' + '
  78. \n' + '
  79. ^ a b Lee, Timothy (11 March 2013). "Major glitch in Bitcoin network sparks sell-off; price temporarily falls 23%". arstechnica.com. Archived from the original on 22 April 2013. Retrieved 15 February 2015.\n' + '
  80. \n' + '
  81. ^ Blagdon, Jeff (12 March 2013). "Technical problems cause Bitcoin to plummet from record high, Mt. Gox suspends deposits". The Verge. Archived from the original on 22 April 2013. Retrieved 12 March 2013.\n' + '
  82. \n' + '
  83. ^ "Bitcoin Charts". Archived from the original on 9 May 2014.\n' + '
  84. \n' + '
  85. ^ Lee, Timothy (20 March 2013). "US regulator Bitcoin Exchanges Must Comply With Money Laundering Laws". Arstechnica. Archived from the original on 21 October 2013. Retrieved 28 July 2017. Bitcoin miners must also register if they trade in their earnings for dollars.\n' + '
  86. \n' + '
  87. ^ "US govt clarifies virtual currency regulatory position". Finextra. 19 March 2013. Archived from the original on 26 March 2014.\n' + '
  88. \n' + '
  89. ^ "Application of FinCEN's Regulations to Persons Administering, Exchanging, or Using Virtual Currencies" (PDF). Department of the Treasury Financial Crimes Enforcement Network. Archived from the original (PDF) on 28 March 2013. Retrieved 19 March 2013.\n' + '
  90. \n' + '
  91. ^ Roose, Kevin (8 April 2013) "Inside the Bitcoin Bubble: BitInstant's CEO – Daily Intelligencer". Archived from the original on 9 April 2014.. Nymag.com. Retrieved on 20 April 2013.\n' + '
  92. \n' + '
  93. ^ "Bitcoin Exchange Rate". Bitcoinscharts.com. Archived from the original on 24 June 2012. Retrieved 15 August 2013.\n' + '
  94. \n' + '
  95. ^ Dillet, Romain. "Feds Seize Assets From Mt. Gox's Dwolla Account, Accuse It Of Violating Money Transfer Regulations". Archived from the original on 9 October 2013. Retrieved 15 May 2013.\n' + '
  96. \n' + '
  97. ^ Berson, Susan A. (2013). "Some basic rules for using 'bitcoin' as virtual money". American Bar Association. Archived from the original on 29 October 2013. Retrieved 26 June 2013.\n' + '
  98. \n' + '
  99. ^ Sampson, Tim (2013). "U.S. government makes its first-ever Bitcoin seizure". The Daily Dot. Archived from the original on 30 June 2013. Retrieved 15 October 2013.\n' + '
  100. \n' + '
  101. ^ a b Ember, Sydney (2 July 2014). "Winner of Bitcoin Auction, Tim Draper, Plans to Expand Currency's Use". The New York Times DealBook. Archived from the original on 28 August 2019. Retrieved 13 June 2019.\n' + '
  102. \n' + '
  103. ^ "After Silk Road seizure, FBI Bitcoin wallet identified and pranked". Archived from the original on 5 April 2014.\n' + '
  104. \n' + '
  105. ^ "Silkroad Seized Coins". Archived from the original on 9 January 2014. Retrieved 2 November 2013.\n' + '
  106. \n' + '
  107. ^ Hill, Kashmir. "The FBI's Plan For The Millions Worth Of Bitcoins Seized From Silk Road". Forbes. Archived from the original on 2 May 2014.\n' + '
  108. \n' + '
  109. ^ Kelion, Leo (18 December 2013). "Bitcoin sinks after China restricts yuan exchanges". bbc.com. BBC. Archived from the original on 19 December 2013. Retrieved 20 December 2013.\n' + '
  110. \n' + '
  111. ^ "China bans banks from bitcoin transactions". The Sydney Morning Herald. Reuters. 6 December 2013. Archived from the original on 23 March 2014. Retrieved 31 October 2014.\n' + '
  112. \n' + '
  113. ^ "Baidu Stops Accepting Bitcoins After China Ban". Bloomberg. New York. 7 December 2013. Archived from the original on 10 December 2013. Retrieved 11 December 2013.\n' + '
  114. \n' + '
  115. ^ "China bars use of virtual money for trading in real goods". English.mofcom.gov.cn. 29 June 2009. Archived from the original on 29 November 2013. Retrieved 10 January 2014.\n' + '
  116. \n' + '
  117. ^ Rayman, Noah (30 July 2014). "You Can Now Donate to Wikipedia in Bitcoin". TIME. Archived from the original on 30 July 2014. Retrieved 27 April 2019.\n' + '
  118. \n' + '
  119. ^ "Bitcoin Core version 0.11.2 released". Bitcoin Project. 13 November 2015. Retrieved 14 November 2016.\n' + '
  120. \n' + '
  121. ^ Kyle Torpey (15 April 2016). "Bitcoin Core 0.12.1 Released, Major Step Forward for Scalability". Bitcoin Magazine. NASDAQ.com. Retrieved 7 November 2016.\n' + '
  122. \n' + '
  123. ^ Antonopoulos, Andreas (2017). Mastering Bitcoin: Programming the Open Blockchain (2nd ed.). O'Reilly Media. ISBN 978-1491954386. BIP-68 and BIP-112 were activated in May 2016 as a soft fork upgrade to the consensus rules.\n' + '
  124. \n' + '
  125. ^ "Bitcoin Core 0.13.1". Bitcoin Core. Retrieved 25 October 2016.\n' + '
  126. \n' + '
  127. ^ "Segregated Witness Benefits". Bitcoin Core. 26 January 2016. Retrieved 20 October 2018.\n' + '
  128. \n' + '
  129. ^ a b c Vigna, Paul (21 July 2017). "Bitcoin Rallies Sharply After Vote Resolves Bitter Scaling Debate". The Wall Street Journal. Archived from the original on 26 January 2020. Retrieved 26 January 2020.\n' + '
  130. \n' + '
  131. ^ Selena Larson (1 August 2017). "Bitcoin split in two, here's what that means". CNN Tech. Cable News Network. Archived from the original on 27 February 2018. Retrieved 2 April 2018.\n' + '
  132. \n' + '
  133. ^ "Bitcoin Hits a New Record High, But Stops Short of $20,000". Fortune. Archived from the original on 14 November 2018. Retrieved 16 April 2019.\n' + '
  134. \n' + '
  135. ^ "RMB Bitcoin trading falls below 1 pct of world total". Xinhuanet. Xinhua. 7 July 2018. Archived from the original on 10 July 2018. Retrieved 10 July 2018.\n' + '
  136. \n' + '
  137. ^ a b "Bitcoin USD". Yahoo Finance!. Archived from the original on 28 January 2019. Retrieved 27 January 2019.\n' + '
  138. \n' + '
  139. ^ Dawkins, David (10 July 2018). "Has CHINA burst the bitcoin BUBBLE? Trading in RMB drops from 90% to 1%". Express. Archived from the original on 10 July 2018. Retrieved 10 July 2018.\n' + '
  140. \n' + '
  141. ^ "Bitcoin turns 10: The obscure technology that became a household name". CNBC. 4 January 2019. Archived from the original on 19 January 2019. Retrieved 18 January 2019.\n' + '
  142. \n' + '
  143. ^ "CVE-2018-17144 Full Disclosure". Bitcoin Core. Retrieved 23 September 2018.\n' + '
  144. \n' + '
  145. ^ Chavez-Dreyfuss, Gertrude (3 July 2018). "Cryptocurrency exchange theft surges in first half of 2018: report". Reuters. Archived from the original on 4 July 2018. Retrieved 10 July 2018.\n' + '
  146. \n' + '
  147. ^ "Cryptocurrencies Tumble After $32 Million South Korea Exchange Hack". Fortune. Bloomberg. 20 July 2018. Archived from the original on 11 July 2018. Retrieved 10 July 2018.\n' + '
  148. \n' + '
  149. ^ Shane, Daniel (11 June 2018). "Billions in cryptocurrency wealth wiped out after hack". CNN. Archived from the original on 11 July 2018. Retrieved 10 July 2018.\n' + '
  150. \n' + '
  151. ^ Russell, Jon (10 July 2018). "The crypto world's latest hack sees Israel's Bancor lose $23.5M". TechCrunch. Archived from the original on 10 July 2018. Retrieved 10 July 2018.\n' + '
  152. \n' + '
  153. ^ Osipovich, Alexander (22 September 2019). "NYSE Owner Launches Long-Awaited Bitcoin Futures". The Wall Street Journal. Archived from the original on 24 September 2019. Retrieved 24 September 2019.\n' + '
  154. \n' + '
  155. ^ Olga Kharif (24 October 2019). "Bakkt Plans First Regulated Options Contracts on Bitcoin Futures". bloomberg.com. Bloomberg. Archived from the original on 24 October 2019. Retrieved 5 November 2019.\n' + '
  156. \n' + '
  157. ^ "YouTube admits error over Bitcoin video purge". bbc.com. BBC. 28 December 2019. Archived from the original on 28 December 2019. Retrieved 28 December 2019.\n' + '
  158. \n' + '
  159. ^ Alexander, Doug (4 February 2019). "Crypto CEO Dies Holding Only Passwords That Can Unlock Millions in Customer Coins". bloomberg.com. Bloomberg. Archived from the original on 16 December 2019. Retrieved 27 January 2020.\n' + '
  160. \n' + '
  161. ^ a b c del Castillo, Michael (19 March 2020). "Bitcoin's Magic Is Fading, And That's A Good Thing". Forbes.com. Archived from the original on 20 March 2020. Retrieved 21 March 2020.\n' + '
  162. \n' + '
  163. ^ "Bitcoin loses half of its value in two-day plunge". CNBC. 13 March 2020. Retrieved 9 December 2020.\n' + '
  164. \n' + '
  165. ^ "MicroStrategy buys $250M in Bitcoin as CEO says it's superior to cash". Washington Business Journal. 11 August 2020. Retrieved 11 August 2020.\n' + '
  166. \n' + '
  167. ^ Business, Oliver Effron, CNN. "Square just bought $50 million in bitcoin". CNN. Retrieved 22 October 2020.\n' + '
  168. \n' + '
  169. ^ "PayPal says all users in US can now buy, hold and sell cryptocurrencies". Techcrunch. 13 November 2020. Retrieved 26 November 2020.\n' + '
  170. \n' + '
  171. ^ "Bitcoin hits an all-time high of just under $20,000". CNN. 30 November 2020. Retrieved 1 December 2020.\n' + '
  172. \n' + '
  173. ^ Catalin Cimpanu (7 December 2020). "BTC-e founder sentenced to five years in prison for laundering ransomware funds". ZDNet. Retrieved 9 December 2020.\n' + '
  174. \n' + '
  175. ^ Olga Kharif (11 December 2020). "169-Year-Old MassMutual Invests $100 Million in Bitcoin". Bloomberg. Retrieved 26 December 2020.\n' + '
  176. \n' + '
  177. ^ Ryan Browne (29 January 2021). "Bitcoin spikes 20% after Elon Musk adds #bitcoin to his Twitter bio". CNBC. Retrieved 2 February 2021.\n' + '
  178. \n' + '
  179. ^ Will Daniel (26 January 2021). "Crypto miner Marathon Patent Group pours $150 million into bitcoin as the token pulls back from record highs". Business Insider. Retrieved 1 February 2021.\n' + '
  180. \n' + '
  181. ^ Yun Li (8 February 2021). "Bitcoin surges above $44,000 to record after Elon Musk's Tesla buys $1.5 billion worth". CNBC. Retrieved 9 February 2021.\n' + '
  182. \n' + '
  183. ^ "Elon Musk says bitcoin is slightly better than holding cash". The Economic Times. Retrieved 19 February 2021.\n' + '
  184. \n' + '
  185. ^ "Canton Zug to accept cryptocurrencies for tax payment beginning in 2021". Canton of Zug. 3 September 2020. Retrieved 24 January 2021.\n' + '
  186. \n' + '
  187. ^ "Kanton Zug akzeptiert Kryptowährungen bei Steuern" (in German). Schweizerischen Radio- und Fernsehgesellschaft. 3 September 2020. Retrieved 24 January 2021.\n' + '
  188. \n' + '
  189. ^ "Secp256k1". Bitcoin Wiki. 24 April 2019. Retrieved 4 February 2021.\n' + '
  190. \n' + '
  191. ^ Knutson, Hans (6 April 2018). "What is the math behind elliptic curve cryptography?". Hacker Noon.\n' + '
  192. \n' + '
  193. ^ Van Hijfte, Stijn (2020). Blockchain Platforms: A Look at the Underbelly of Distributed Platforms. Morgan & Claypool Publishers. ISBN 9781681738925.\n' + '
  194. \n' + '
  195. ^ Romain Dillet (9 August 2013). "Bitcoin Ticker Available On Bloomberg Terminal For Employees". TechCrunch. Archived from the original on 1 November 2014. Retrieved 2 November 2014.\n' + '
  196. \n' + '
  197. ^ "Bitcoin Composite Quote (XBT)". CNN Money. CNN. Archived from the original on 27 October 2014. Retrieved 2 November 2014.\n' + '
  198. \n' + '
  199. ^ "XBT – Bitcoin". xe.com. Archived from the original on 2 November 2014. Retrieved 2 November 2014.\n' + '
  200. \n' + '
  201. ^ "Regulation of Bitcoin in Selected Jurisdictions" (PDF). The Law Library of Congress, Global Legal Research Center. January 2014. Archived (PDF) from the original on 14 October 2014. Retrieved 26 August 2014.\n' + '
  202. \n' + '
  203. ^ Katie Pisa & Natasha Maguder (9 July 2014). "Bitcoin your way to a double espresso". cnn.com. CNN. Archived from the original on 18 June 2015. Retrieved 23 April 2015.\n' + '
  204. \n' + '
  205. ^ a b "Blockchair". Blockchair.com. Archived from the original on 13 October 2019. Retrieved 13 October 2019.\n' + '
  206. \n' + '
  207. ^ a b c d e "Charts". Blockchain.info. Archived from the original on 3 November 2014. Retrieved 2 November 2014.\n' + '
  208. \n' + '
  209. ^ a b c d e f "The great chain of being sure about things". The Economist. The Economist Newspaper Limited. 31 October 2015. Archived from the original on 3 July 2016. Retrieved 3 July 2016.\n' + '
  210. \n' + '
  211. ^ Sparkes, Matthew (9 June 2014). "The coming digital anarchy". The Daily Telegraph. London. Archived from the original on 23 January 2015. Retrieved 7 January 2015.\n' + '
  212. \n' + '
  213. ^ a b Ashlee Vance (14 November 2013). "2014 Outlook: Bitcoin Mining Chips, a High-Tech Arms Race". Businessweek. Archived from the original on 21 November 2013. Retrieved 24 November 2013.\n' + '
  214. \n' + '
  215. ^ Ritchie S. King; Sam Williams; David Yanofsky (17 December 2013). "By reading this article, you're mining bitcoins". qz.com. Atlantic Media Co. Archived from the original on 17 December 2013. Retrieved 17 December 2013.\n' + '
  216. \n' + '
  217. ^ Shin, Laura (24 May 2016). "Bitcoin Production Will Drop By Half In July, How Will That Affect The Price?". Forbes. Archived from the original on 24 May 2016. Retrieved 13 July 2016.\n' + '
  218. \n' + '
  219. ^ a b c d e Joshua A. Kroll; Ian C. Davey; Edward W. Felten (11–12 June 2013). "The Economics of Bitcoin Mining, or Bitcoin in the Presence of Adversaries" (PDF). The Twelfth Workshop on the Economics of Information Security (WEIS 2013). Archived (PDF) from the original on 9 May 2016. Retrieved 26 April 2016. A transaction fee is like a tip or gratuity left for the miner.\n' + '
  220. \n' + '
  221. ^ Orcutt, Mike (19 May 2015). "Leaderless Bitcoin Struggles to Make Its Most Crucial Decision". MIT Technology Review. Archived from the original on 18 October 2017. Retrieved 22 June 2017.\n' + '
  222. \n' + '
  223. ^ "Man Throws Away 7,500 Bitcoins, Now Worth $7.5 Million". CBS DC. 29 November 2013. Archived from the original on 15 January 2014. Retrieved 23 January 2014.\n' + '
  224. \n' + '
  225. ^ Krause, Elliott (5 July 2018). "A Fifth of All Bitcoin Is Missing. These Crypto Hunters Can Help". The Wall Street Journal. Archived from the original on 9 July 2018. Retrieved 8 July 2018.\n' + '
  226. \n' + '
  227. ^ Jeffries, Adrianne (19 December 2013). "How to steal Bitcoin in three easy steps". The Verge. Archived from the original on 27 July 2019. Retrieved 17 January 2014.\n' + '
  228. \n' + '
  229. ^ a b Harney, Alexandra; Stecklow, Steve (16 November 2017). "Twice burned - How Mt. Gox's bitcoin customers could lose again". Reuters. Archived from the original on 29 August 2019. Retrieved 6 September 2018.\n' + '
  230. \n' + '
  231. ^ Martindale, Jon (16 March 2018). "Who owns all the Bitcoin? A few billionaire whales in a small pond". Digital Trends. Archived from the original on 30 June 2019. Retrieved 1 July 2019.\n' + '
  232. \n' + '
  233. ^ "Bitcoin mania is hurting PC gamers by pushing up GPU prices". Archived from the original on 2 February 2018. Retrieved 2 February 2018.\n' + '
  234. \n' + '
  235. ^ "Cryptocurrency mining operation launched by Iron Bridge Resources". World Oil. 26 January 2018. Archived from the original on 30 January 2018.\n' + '
  236. \n' + '
  237. ^ a b Andolfatto, David (31 March 2014). "Bitcoin and Beyond: The Possibilities and Pitfalls of Virtual Currencies" (PDF). Dialogue with the Fed. Federal Reserve Bank of St. Louis. Archived (PDF) from the original on 9 April 2014. Retrieved 16 April 2014.\n' + '
  238. \n' + '
  239. ^ Sherman, Alan; Javani, Farid; Golaszewski, Enis (25 March 2019). "On the Origins and Variations of Blockchain Technologies". IEEE Security and Policy. 17 (1): 72–77. arXiv:1810.06130. doi:10.1109/MSEC.2019.2893730. S2CID 53114747.\n' + '
  240. \n' + '
  241. ^ "Difficulty History" (The ratio of all hashes over valid hashes is D x 4,295,032,833, where D is the published "Difficulty" figure.). Blockchain.info. Archived from the original on 8 April 2015. Retrieved 26 March 2015.\n' + '
  242. \n' + '
  243. ^ Hampton, Nikolai (5 September 2016). "Understanding the blockchain hype: Why much of it is nothing more than snake oil and spin". Computerworld. IDG. Archived from the original on 6 September 2016. Retrieved 5 September 2016.\n' + '
  244. \n' + '
  245. ^ Biggs, John (8 April 2013). "How To Mine Bitcoins". Techcrunch. Archived from the original on 6 July 2017.\n' + '
  246. \n' + '
  247. ^ Skudnov, Rostislav (2012). Bitcoin Clients (PDF) (Bachelor's Thesis). Turku University of Applied Sciences. Archived (PDF) from the original on 18 January 2014. Retrieved 16 January 2014.\n' + '
  248. \n' + '
  249. ^ "Bitcoin Core version 0.9.0 released". bitcoin.org. Archived from the original on 27 February 2015. Retrieved 8 January 2015.\n' + '
  250. \n' + '
  251. ^ Metz, Cade (19 August 2015). "The Bitcoin Schism Shows the Genius of Open Source". Wired. Condé Nast. Archived from the original on 30 June 2016. Retrieved 3 July 2016.\n' + '
  252. \n' + '
  253. ^ Allison, Ian (28 April 2017). "Ethereum co-founder Dr Gavin Wood and company release Parity Bitcoin". International Business Times. Archived from the original on 28 April 2017. Retrieved 28 April 2017.\n' + '
  254. \n' + '
  255. ^ Adam Serwer & Dana Liebelson (10 April 2013). "Bitcoin, Explained". motherjones.com. Mother Jones. Archived from the original on 27 April 2014. Retrieved 26 April 2014.\n' + '
  256. \n' + '
  257. ^ "Bitcoin: Bitcoin under pressure". The Economist. 30 November 2013. Archived from the original on 30 November 2013. Retrieved 30 November 2013.\n' + '
  258. \n' + '
  259. ^ "Blockchain Size". Blockchain.info. Archived from the original on 27 May 2017. Retrieved 16 January 2018.\n' + '
  260. \n' + '
  261. ^ Gervais, Arthur; O. Karame, Ghassan; Gruber, Damian; Capkun, Srdjan. "On the Privacy Provisions of Bloom Filters in Lightweight Bitcoin Clients" (PDF). Archived (PDF) from the original on 5 October 2016. Retrieved 3 September 2016.\n' + '
  262. \n' + '
  263. ^ Bill Barhydt (4 June 2014). "3 reasons Wall Street can't stay away from bitcoin". NBCUniversal. Archived from the original on 3 April 2015. Retrieved 2 April 2015.\n' + '
  264. \n' + '
  265. ^ "MtGox gives bankruptcy details". bbc.com. BBC. 4 March 2014. Archived from the original on 12 March 2014. Retrieved 13 March 2014.\n' + '
  266. \n' + '
  267. ^ a b Barski, Conrad; Wilmer, Chris (2015). Bitcoin for the Befuddled. No Starch Press. ISBN 978-1-59327-573-0.\n' + '
  268. \n' + '
  269. ^ Popper, Nathaniel (19 December 2017). "How the Winklevoss Twins Found Vindication in a Bitcoin Fortune". The New York Times. Archived from the original on 20 June 2019. Retrieved 19 June 2019.\n' + '
  270. \n' + '
  271. ^ Alexander, Doug (15 February 2019). "Quadriga's late founder used to store clients' Bitcoin passwords on paper so they wouldn't get lost". Bloomberg News. Financial Post. Archived from the original on 20 June 2019. Retrieved 19 June 2019.\n' + '
  272. \n' + '
  273. ^ a b Staff, Verge (13 December 2013). "Casascius, maker of shiny physical bitcoins, shut down by Treasury Department". The Verge. Archived from the original on 10 January 2014. Retrieved 10 January 2014.\n' + '
  274. \n' + '
  275. ^ a b c d e Ahonen, Elias; Rippon, Matthew J.; Kesselman, Howard (15 April 2016). Encyclopedia of Physical Bitcoins and Crypto-Currencies. ISBN 978-0-9950-8990-7.\n' + '
  276. \n' + '
  277. ^ Mack, Eric (25 October 2011). "Are physical Bitcoins legal?". CNET. Archived from the original on 26 June 2019. Retrieved 19 May 2019.\n' + '
  278. \n' + '
  279. ^ British Museum (2012). "Bitcoin token with digital code for bitcoin currency". Retrieved 17 May 2019.\n' + '
  280. \n' + '
  281. ^ Roberts, Daniel (15 December 2017). "How to send bitcoin to a hardware wallet". Yahoo Finance. Archived from the original on 17 February 2018. Retrieved 17 February 2018.\n' + '
  282. \n' + '
  283. ^ Meola, Andrew (5 October 2017). "How distributed ledger technology will change the way the world works". Business Insider. Archived from the original on 27 April 2018. Retrieved 26 July 2018.\n' + '
  284. \n' + '
  285. ^ Jerry Brito & Andrea Castillo (2013). "Bitcoin: A Primer for Policymakers" (PDF). Mercatus Center. George Mason University. Archived (PDF) from the original on 21 September 2013. Retrieved 22 October 2013.\n' + '
  286. \n' + '
  287. ^ Tschorsch, Florian; Scheuermann, Björn (2016). "Bitcoin and Beyond: A Technical Survey on Decentralized Digital Currencies". IEEE Communications Surveys & Tutorials. 18 (3): 2084–2123. doi:10.1109/comst.2016.2535718. S2CID 5115101.\n' + '
  288. \n' + '
  289. ^ Beikverdi, A.; Song, J. (June 2015). Trend of centralization in Bitcoin's distributed network. 2015 IEEE/ACIS 16th International Conference on Software Engineering, Artificial Intelligence, Networking and Parallel/Distributed Computing (SNPD). pp. 1–6. doi:10.1109/SNPD.2015.7176229. ISBN 978-1-4799-8676-7. S2CID 15516195.\n' + '
  290. \n' + '
  291. ^ a b c Gervais, Arthur; Karame, Ghassan O.; Capkun, Vedran; Capkun, Srdjan. "Is Bitcoin a Decentralized Currency?". InfoQ. InfoQ & IEEE Computer Society. Archived from the original on 10 October 2016. Retrieved 11 October 2016.\n' + '
  292. \n' + '
  293. ^ Wilhelm, Alex. "Popular Bitcoin Mining Pool Promises To Restrict Its Compute Power To Prevent Feared '51%' Fiasco". TechCrunch. Archived from the original on 5 December 2017. Retrieved 25 January 2018.\n' + '
  294. \n' + '
  295. ^ Chan, Edwin. "China Plans to Ban Cryptocurrency Mining in Renewed Clampdown". www.bloomberg.com. Archived from the original on 18 December 2019. Retrieved 10 April 2019. While China was once home to about 70 percent of Bitcoin mining and 90 percent of trades, authorities have waged a nearly two-year campaign to shrink the crypto industry amid concerns over speculative bubbles, fraud and wasteful energy consumption.\n' + '
  296. \n' + '
  297. ^ Simonite, Tom (5 September 2013). "Mapping the Bitcoin Economy Could Reveal Users' Identities". MIT Technology Review. Retrieved 2 April 2014.\n' + '
  298. \n' + '
  299. ^ Lee, Timothy (21 August 2013). "Five surprising facts about Bitcoin". The Washington Post. Archived from the original on 12 October 2013. Retrieved 2 April 2014.\n' + '
  300. \n' + '
  301. ^ McMillan, Robert (6 June 2013). "How Bitcoin lets you spy on careless companies". wired.co.uk. Conde Nast. Archived from the original on 9 February 2014. Retrieved 3 April 2020.\n' + '
  302. \n' + '
  303. ^ Ben-Sasson, Eli; Chiesa, Alessandro; Garman, Christina; Green, Matthew; Miers, Ian; Tromer, Eran; Virza, Madars (2014). "Zerocash: Decentralized Anonymous Payments from Bitcoin" (PDF). 2014 IEEE Symposium on Security and Privacy. IEEE computer society. Archived (PDF) from the original on 14 October 2014. Retrieved 31 October 2014.\n' + '
  304. \n' + '
  305. ^ Möser, Malte; Böhme, Rainer; Breuker, Dominic (2013). An Inquiry into Money Laundering Tools in the Bitcoin Ecosystem (PDF). 2013 APWG eCrime Researchers Summit. IEEE. ISBN 978-1-4799-1158-5. Archived (PDF) from the original on 26 June 2018. Retrieved 19 June 2019.\n' + '
  306. \n' + '
  307. ^ a b Feuer, Alan (14 December 2013). "The Bitcoin Ideology". The New York Times. Archived from the original on 1 July 2018. Retrieved 1 July 2018.\n' + '
  308. \n' + '
  309. ^ Friedrich von Hayek (October 1976). Denationalisation of Money: The Argument Refined. 2 Lord North Street, Westminster, London SWIP 3LB: The institute of economic affairs. ISBN 978-0-255-36239-9. Archived from the original on 11 January 2020. Retrieved 10 September 2015.CS1 maint: location (link)\n' + '
  310. \n' + '
  311. ^ European Central Bank (October 2012). Virtual Currency Schemes (PDF). Frankfurt am Main: European Central Bank. ISBN 978-92-899-0862-7. Archived (PDF) from the original on 6 November 2012.\n' + '
  312. \n' + '
  313. ^ a b "Bitcoin and other cryptocurrencies are useless". The Economist. 30 August 2018. Archived from the original on 4 September 2018. Retrieved 4 September 2018. Lack of adoption and loads of volatility mean that cryptocurrencies satisfy none of those criteria.\n' + '
  314. \n' + '
  315. ^ a b Krugman, Paul (29 January 2018). "Bubble, Bubble, Fraud and Trouble". The New York Times. Archived from the original on 4 June 2018.\n' + '
  316. \n' + '
  317. ^ a b Tourianski, Julia. "The Declaration Of Bitcoin's Independence". Archive.org. Archived from the original on 23 March 2019. Retrieved 1 July 2018.\n' + '
  318. \n' + '
  319. ^ a b Dodd, Nigel (2017). "The social life of Bitcoin" (PDF). LSE Research Online. Archived (PDF) from the original on 1 July 2018. Retrieved 1 July 2018.\n' + '
  320. \n' + '
  321. ^ Golumbia, David (2015). Lovink, Geert (ed.). Bitcoin as Politics: Distributed Right-Wing Extremism. Institute of Network Cultures, Amsterdam. pp. 117–131. ISBN 978-90-822345-5-8. SSRN 2589890.\n' + '
  322. \n' + '
  323. ^ Peters, Jeremy W.; Popper, Nathaniel (14 June 2018). "Stephen Bannon Buys Into Bitcoin". The New York Times. Archived from the original on 1 July 2018. Retrieved 1 July 2018.\n' + '
  324. \n' + '
  325. ^ a b Matthew Graham Wilson & Aaron Yelowitz (November 2014). "Characteristics of Bitcoin Users: An Analysis of Google Search Data". Social Science Research Network. Working Papers Series. SSRN 2518603.\n' + '
  326. \n' + '
  327. ^ a b "Monetarists Anonymous". The Economist. The Economist Newspaper Limited. 29 September 2012. Archived from the original on 20 October 2013. Retrieved 21 October 2013.\n' + '
  328. \n' + '
  329. ^ a b "The magic of mining". The Economist. 13 January 2015. Archived from the original on 12 January 2015. Retrieved 13 January 2015.\n' + '
  330. \n' + '
  331. ^ a b "Free Exchange. Money from nothing. Chronic deflation may keep Bitcoin from displacing its rivals". The Economist. 15 March 2014. Archived from the original on 25 March 2014. Retrieved 25 March 2014.\n' + '
  332. \n' + '
  333. ^ a b Shiller, Robert (1 March 2014). "In Search of a Stable Electronic Currency". The New York Times. Archived from the original on 24 October 2014.\n' + '
  334. \n' + '
  335. ^ a b Murphy, Hannah (8 June 2018). "Who really owns bitcoin now?". Financial Times. Archived from the original on 10 June 2018. Retrieved 10 June 2018.\n' + '
  336. \n' + '
  337. ^ Karkaria, Urvaksh (23 September 2014). "Atlanta-based BitPay hooks up with PayPal to expand bitcoin adoption". Atlanta Business Chronicle. Archived from the original on 26 October 2014.\n' + '
  338. \n' + '
  339. ^ Katz, Lily (12 July 2017). "Bitcoin Acceptance Among Retailers Is Low and Getting Lower". Bloomberg. Archived from the original on 25 January 2018. Retrieved 25 January 2018.\n' + '
  340. \n' + '
  341. ^ Kharif, Olga (1 August 2018). "Bitcoin's Use in Commerce Keeps Falling Even as Volatility Eases". Bloomberg. Archived from the original on 2 August 2018. Retrieved 2 August 2018.\n' + '
  342. \n' + '
  343. ^ Chan, Bernice (16 January 2015). "Bitcoin transactions cut the cost of international money transfers". South China Morning Post. Archived from the original on 31 May 2019. Retrieved 31 May 2019.\n' + '
  344. \n' + '
  345. ^ "Bitcoin firms dumped by National Australia Bank as 'too risky'". The Guardian. Australian Associated Press. 10 April 2014. Archived from the original on 23 February 2015. Retrieved 23 February 2015.\n' + '
  346. \n' + '
  347. ^ Weir, Mike (1 December 2014). "HSBC severs links with firm behind Bitcoin fund". bbc.com. BBC. Archived from the original on 3 February 2015. Retrieved 9 January 2015.\n' + '
  348. \n' + '
  349. ^ "ACCC investigating why banks are closing bitcoin companies' accounts". Financial Review. Archived from the original on 11 February 2016. Retrieved 28 January 2016.\n' + '
  350. \n' + '
  351. ^ "Bitcoin futures surge in first day of trading". CBS News. 11 December 2017. Archived from the original on 31 May 2019. Retrieved 31 May 2019.\n' + '
  352. \n' + '
  353. ^ "Chicago Mercantile Exchange jumps into bitcoin futures". CBS News. 18 December 2017. Archived from the original on 31 May 2019. Retrieved 31 May 2019.\n' + '
  354. \n' + '
  355. ^ "Venezuela Has Bitcoin Stash and Doesn't Know What to Do With It". Bloomberg. 26 September 2019. Archived from the original on 26 September 2019. Retrieved 26 September 2019.\n' + '
  356. \n' + '
  357. ^ Lee, Timothy B. "The $11 million in bitcoins the Winklevoss brothers bought is now worth $32 million". The Switch. The Washington Post. Archived from the original on 6 July 2017. Retrieved 11 August 2017.\n' + '
  358. \n' + '
  359. ^ "Jersey approve Bitcoin fund launch on island". BBC news. 10 July 2014. Archived from the original on 10 July 2014. Retrieved 10 July 2014.\n' + '
  360. \n' + '
  361. ^ Hill, Kashmir. "How You Should Have Spent $100 In 2013 (Hint: Bitcoin)". Forbes. Archived from the original on 19 February 2015. Retrieved 16 February 2015.\n' + '
  362. \n' + '
  363. ^ Steverman, Ben (23 December 2014). "The Best and Worst Investments of 2014". bloomberg.com. Bloomberg LP. Archived from the original on 9 January 2015. Retrieved 9 January 2015.\n' + '
  364. \n' + '
  365. ^ Gilbert, Mark (29 December 2015). "Bitcoin Won 2015. Apple ... Did Not". Bloomberg. Archived from the original on 29 December 2015. Retrieved 29 December 2015.\n' + '
  366. \n' + '
  367. ^ "Top 100 Richest Bitcoin Addresses and Bitcoin distribution". bitinfocharts.com. Archived from the original on 15 October 2017. Retrieved 14 October 2017.\n' + '
  368. \n' + '
  369. ^ "MicroStrategy Buys $50 Million Worth Of Bitcoin, Topping Up Holdings To $766M".\n' + '
  370. \n' + '
  371. ^ "MicroStrategy buys $250M in Bitcoin as CEO says it's superior to cash". Washington Business Journal. 11 August 2020. Retrieved 11 August 2020.\n' + '
  372. \n' + '
  373. ^ Wilson, Tom (18 May 2021). "Buyers beware as "altcoin" frenzy bruises bitcoin". Reuters. Retrieved 19 May 2021.\n' + '
  374. \n' + '
  375. ^ Simonite, Tom (12 June 2013). "Bitcoin Millionaires Become Investing Angels". Computing News. MIT Technology Review. Retrieved 13 June 2013.\n' + '
  376. \n' + '
  377. ^ a b Robin Sidel (1 December 2014). "Ten-hut! Bitcoin Recruits Snap To". The Wall Street Journal. Dow Jones & Company. Retrieved 9 December 2014.[dead link]\n' + '
  378. \n' + '
  379. ^ Alex Hern (1 July 2014). "Silk Road's legacy 30,000 bitcoin sold at auction to mystery buyers". The Guardian. Archived from the original on 23 October 2014. Retrieved 31 October 2014.\n' + '
  380. \n' + '
  381. ^ "CoinSeed raises $7.5m, invests $5m in Bitcoin mining hardware – Investment Round Up". Red Herring. 24 January 2014. Archived from the original on 9 March 2014. Retrieved 9 March 2014.\n' + '
  382. \n' + '
  383. ^ a b Tasca, Paolo (7 September 2015), Digital Currencies: Principles, Trends, Opportunities, and Risks, Social Science Research Network, SSRN 2657598\n' + '
  384. \n' + '
  385. ^ Moore, Heidi (3 April 2013). "Confused about Bitcoin? It's 'the Harlem Shake of currency'". The Guardian. Archived from the original on 1 March 2014. Retrieved 2 May 2014.\n' + '
  386. \n' + '
  387. ^ Lee, Timothy (5 November 2013). "When will the people who called Bitcoin a bubble admit they were wrong". The Washington Post. Archived from the original on 11 January 2014. Retrieved 10 January 2014.\n' + '
  388. \n' + '
  389. ^ Liu, Alec (19 March 2013). "When Governments Take Your Money, Bitcoin Looks Really Good". Motherboard. Archived from the original on 7 February 2014. Retrieved 7 January 2014.\n' + '
  390. \n' + '
  391. ^ Ben Rooney (29 November 2013). "Bitcoin worth almost as much as gold". CNN. Archived from the original on 26 October 2014. Retrieved 31 October 2014.\n' + '
  392. \n' + '
  393. ^ "Bitcoin prices remain below $600 amid bearish chart signals". nasdaq.com. August 2014. Archived from the original on 14 October 2014. Retrieved 31 October 2014..\n' + '
  394. \n' + '
  395. ^ Williams, Mark T. (21 October 2014). "Virtual Currencies – Bitcoin Risk" (PDF). World Bank Conference Washington DC. Boston University. Archived (PDF) from the original on 11 November 2014. Retrieved 11 November 2014.\n' + '
  396. \n' + '
  397. ^ Irrera, Tom Wilson, Anna (11 January 2021). "Analysis: Cancel your weekends! Bitcoin doesn't rest, and neither can you". Reuters. Archived from the original on 6 February 2021. Trading volumes across six major cryptocurrency exchanges have been 10% higher at weekends than weekdays in that period\n' + '
  398. \n' + '
  399. ^ Zaki, Myret (14 January 2021). "Bitcoin: The Derivative Bomb". The Market (in German). Archived from the original on 15 January 2021. the lion’s share of institutional trading in bitcoin is being done without owning any single bitcoin. The bitcoin derivative boom was encouraged by the fact that you can get 2 to 3 times leverage on the CME, and more than 100 x leverage on native crypto derivative exchanges. In 2020, the ratio between bitcoin futures and spot volumes has increased from 2,3 to 4,6 in 2019\n' + '
  400. \n' + '
  401. ^ Bambrough, Billy (30 January 2021). "Data Reveals Bitcoin Could Be About To Become The New GameStop After Huge Price Spike". Forbes. Archived from the original on 9 February 2021. Retrieved 9 February 2021. The net short position in bitcoin futures is now the biggest it has ever been, according to the CFTC's latest Traders in Financial Futures report. .. hedge funds increasingly bet against the bitcoin price\n' + '
  402. \n' + '
  403. ^ "CFTC Commitments of Traders Short Report - Financial Traders in Markets (Futures Only)". cftc.gov. Commodity Futures Trading Commission (CFTC). 2–5 February 2021. Archived from the original on 8 February 2021. BITCOIN - CHICAGO MERCANTILE EXCHANGE, CFTC Code #133741\n' + '
  404. \n' + '
  405. ^ "WSJ Markets Bitcoin USD". Retrieved 8 February 2021.\n' + '
  406. \n' + '
  407. ^ "China May Be Gearing Up to Ban Bitcoin". pastemagazine.com. Archived from the original on 3 October 2017. Retrieved 6 October 2017. The decentralized nature of bitcoin is such that it is impossible to "ban" the cryptocurrency, but if you shut down exchanges and the peer-to-peer economy running on bitcoin, it's a de facto ban.\n' + '
  408. \n' + '
  409. ^ "Regulation of Cryptocurrency Around the World" (PDF). Library of Congress. The Law Library of Congress, Global Legal Research Center. June 2018. pp. 4–5. Archived (PDF) from the original on 14 August 2018. Retrieved 15 August 2018.\n' + '
  410. \n' + '
  411. ^ a b "Iran: New Crypto Law Requires Selling Bitcoin Directly to Central Bank to Fund Imports". aawsat.com. 31 October 2020. Retrieved 1 November 2020.\n' + '
  412. \n' + '
  413. ^ "Iran Is Pivoting to Bitcoin". vice.com. 3 November 2020. Retrieved 10 November 2020.\n' + '
  414. \n' + '
  415. ^ "Iran Has a Bitcoin Strategy to Beat Trump". foreignpolicy.com. 24 January 2020. Retrieved 10 November 2020.\n' + '
  416. \n' + '
  417. ^ "Customer Advisory: Use Caution When Buying Digital Coins or Tokens" (PDF). U.S.Commodity Futures Trading Commission. Archived (PDF) from the original on 17 July 2018. Retrieved 17 July 2018.\n' + '
  418. \n' + '
  419. ^ "Investor Alert: Bitcoin and Other Virtual Currency-related Investments". Investor.gov. U.S. Securities and Exchange Commission. 7 May 2014. Archived from the original on 30 June 2018. Retrieved 17 July 2018.\n' + '
  420. \n' + '
  421. ^ "Ponzi schemes Using virtual Currencies" (PDF). sec.gov. U.S. Securities and Exchange Commission. Archived (PDF) from the original on 16 June 2018. Retrieved 17 July 2018.\n' + '
  422. \n' + '
  423. ^ "Warning to consumers on virtual currencies" (PDF). European Banking Authority. 12 December 2013. Archived from the original (PDF) on 28 December 2013. Retrieved 23 December 2013.\n' + '
  424. \n' + '
  425. ^ "Investor Alerts Don't Fall for Cryptocurrency-Related Stock Scams". FINRA.org. Financial Industry Regulatory Authority. 21 December 2017. Archived from the original on 1 July 2018. Retrieved 23 July 2018.\n' + '
  426. \n' + '
  427. ^ "Informed Investor Advisory: Cryptocurrencies". North American Securities Administrators Association. April 2018. Archived from the original on 23 July 2018. Retrieved 23 July 2018.\n' + '
  428. \n' + '
  429. ^ Dean, James (25 May 2018). "Bitcoin investigation to focus on British traders, US officials examine manipulation of cryptocurrency prices". The Times. Archived from the original on 25 May 2018. Retrieved 25 May 2018.\n' + '
  430. \n' + '
  431. ^ Cornish, Chloe (24 May 2018). "Bitcoin slips again on reports of US DoJ investigation". Financial Times. Archived from the original on 24 May 2018. Retrieved 24 May 2018.\n' + '
  432. \n' + '
  433. ^ Robinson, Matt; Schoenberg, Tom (24 May 2018). "U.S. Launches Criminal Probe into Bitcoin Price Manipulation". Bloomberg. Archived from the original on 24 May 2018. Retrieved 24 May 2018.\n' + '
  434. \n' + '
  435. ^ McCoy, Kevin (24 May 2018). "Bitcoin value gyrates amid report of Department of Justice manipulation investigation". USA Today. Archived from the original on 24 May 2018. Retrieved 25 May 2018.\n' + '
  436. \n' + '
  437. ^ Rubin, Gabriel T.; Michaels, Dave; Osipovich, Alexander (8 June 2018). "U.S. regulators demand trading data from bitcoin exchanges in manipulation probe". MarketWatch. Archived from the original on 8 June 2018. Retrieved 9 June 2018. Note:this is a short open access version of a Wall Street Journal article\n' + '
  438. \n' + '
  439. ^ Rubin, Gabriel T.; Michaels, Dave; Osipovich, Alexander (8 June 2018). "U.S. regulators demand trading data from bitcoin exchanges in manipulation probe". The Wall Street Journal. Archived from the original on 8 June 2018. Retrieved 9 June 2018. (paywalled)\n' + '
  440. \n' + '
  441. ^ Fung, Brian (21 May 2018). "State regulators unveil nationwide crackdown on suspicious cryptocurrency investment schemes". The Washington Post. Archived from the original on 27 May 2018. Retrieved 27 May 2018.\n' + '
  442. \n' + '
  443. ^ Gandal, Neil; Hamrick, J.T.; Moore, Tyler; Oberman, Tali (May 2018). "Price manipulation in the Bitcoin ecosystem". Journal of Monetary Economics. 95: 86–96. doi:10.1016/j.jmoneco.2017.12.004. S2CID 26358036.\n' + '
  444. \n' + '
  445. ^ Lee, Tim (12 December 2017). "A brief history of Bitcoin hacks and frauds". Ars Technica. Archived from the original on 28 May 2018. Retrieved 27 May 2018.\n' + '
  446. \n' + '
  447. ^ Griffin, John M.; Shams, Amin (13 June 2018). "Is Bitcoin Really Un-Tethered?". Social Science Research Network. SSRN 3195066.\n' + '
  448. \n' + '
  449. ^ Popper, Nathaniel (13 June 2018). "Bitcoin's Price Was Artificially Inflated Last Year, Researchers Say". The New York Times. Archived from the original on 13 June 2018. Retrieved 13 June 2018.\n' + '
  450. \n' + '
  451. ^ Shaban, Hamza (14 June 2018). "Bitcoin's astronomical rise last year was buoyed by market manipulation, researchers say". The Washington Post. Archived from the original on 15 June 2018. Retrieved 14 June 2018.\n' + '
  452. \n' + '
  453. ^ a b Janda, Michael (18 June 2018). "Cryptocurrencies like bitcoin cannot replace money, says Bank for International Settlements". ABC (Australia). Archived from the original on 18 June 2018. Retrieved 18 June 2018.\n' + '
  454. \n' + '
  455. ^ Hyun Song Shin (June 2018). "Chapter V. Cryptocurrencies: looking beyond the hype" (PDF). BIS 2018 Annual Economic Report. Bank for International Settlements. Archived (PDF) from the original on 18 June 2018. Retrieved 19 June 2018. Put in the simplest terms, the quest for decentralised trust has quickly become an environmental disaster.\n' + '
  456. \n' + '
  457. ^ Hiltzik, Michael (18 June 2018). "Is this scathing report the death knell for bitcoin?". Los Angeles Times. Archived from the original on 18 June 2018. Retrieved 19 June 2018.\n' + '
  458. \n' + '
  459. ^ Velde, François (December 2013). "Bitcoin: A primer" (PDF). Chicago Fed letter. Federal Reserve Bank of Chicago. p. 4. Archived (PDF) from the original on 26 October 2014. Retrieved 3 September 2016.\n' + '
  460. \n' + '
  461. ^ Wile, Rob (6 April 2014). "St. Louis Fed Economist: Bitcoin Could Be A Good Threat To Central Banks". businessinsider.com. Business Insider. Archived from the original on 24 September 2015. Retrieved 16 April 2014.\n' + '
  462. \n' + '
  463. ^ Andolfatto, David (24 December 2013). "In gold we trust?". MacroMania. David Andolfatto. Archived from the original on 12 April 2017. Retrieved 17 April 2014. Also, note that I am not against gold or bitcoin (or whatever) as a currency. In fact, I think that the threat that they pose as alternate currency can serve as a useful check on a central bank.\n' + '
  464. \n' + '
  465. ^ Costelloe, Kevin (29 November 2017). "Bitcoin 'Ought to Be Outlawed,' Nobel Prize Winner Stiglitz Says". Bloomberg. Archived from the original on 12 June 2018. Retrieved 5 June 2018. It doesn't serve any socially useful function.\n' + '
  466. \n' + '
  467. ^ "Economics Nobel prize winner, Richard Thaler: "The market that looks most like a bubble to me is Bitcoin and its brethren"". ECO Portuguese Economy. 22 January 2018. Archived from the original on 12 June 2018. Retrieved 7 June 2018.\n' + '
  468. \n' + '
  469. ^ a b Wolff-Mann, Ethan (27 April 2018). "'Only good for drug dealers': More Nobel prize winners snub bitcoin". Yahoo Finance. Archived from the original on 12 June 2018. Retrieved 7 June 2018.\n' + '
  470. \n' + '
  471. ^ "Bitcoin biggest bubble in history, says economist who predicted 2008 crash". Archived from the original on 12 June 2018.\n' + '
  472. \n' + '
  473. ^ Braue, David (11 March 2014). "Bitcoin confidence game is a Ponzi scheme for the 21st century". ZDNet. Archived from the original on 6 October 2016. Retrieved 5 October 2016.\n' + '
  474. \n' + '
  475. ^ Clinch, Matt (10 March 2014). "Roubini launches stinging attack on bitcoin". CNBC. Archived from the original on 6 October 2014. Retrieved 2 July 2014.\n' + '
  476. \n' + '
  477. ^ "This Billionaire Just Called Bitcoin a 'Pyramid Scheme'". Archived from the original on 24 September 2017. Retrieved 23 September 2017.\n' + '
  478. \n' + '
  479. ^ Ott Ummelas & Milda Seputyte (31 January 2014). "Bitcoin 'Ponzi' Concern Sparks Warning From Estonia Bank". bloomberg.com. Bloomberg. Archived from the original on 29 March 2014. Retrieved 1 April 2014.\n' + '
  480. \n' + '
  481. ^ Posner, Eric (11 April 2013). "Fool's Gold: Bitcoin is a Ponzi scheme—the Internet's favorite currency will collapse". Slate. Archived from the original on 26 March 2014. Retrieved 1 April 2014.\n' + '
  482. \n' + '
  483. ^ Kaushik Basu (July 2014). "Ponzis: The Science and Mystique of a Class of Financial Frauds" (PDF). World Bank Group. Archived (PDF) from the original on 31 October 2014. Retrieved 30 October 2014.\n' + '
  484. \n' + '
  485. ^ "Federal Council report on virtual currencies in response to the Schwaab (13.3687) and Weibel (13.4070) postulates" (PDF). Federal Council (Switzerland). Swiss Confederation. 25 June 2014. Archived (PDF) from the original on 5 December 2014. Retrieved 28 November 2014.\n' + '
  486. \n' + '
  487. ^ Mooney, Chris; Mufson, Steven (19 December 2017). "Why the bitcoin craze is using up so much energy". The Washington Post. Archived from the original on 9 January 2018. Retrieved 11 January 2018. several experts told The Washington Post that bitcoin probably uses as much as 1 to 4 gigawatts, or billion watts, of electricity, roughly the output of one to three nuclear reactors.\n' + '
  488. \n' + '
  489. ^ a b Roberts, Paul (9 March 2018). "This Is What Happens When Bitcoin Miners Take Over Your Town - Eastern Washington had cheap power and tons of space. Then the suitcases of cash started arriving". Politico. Archived from the original on 9 March 2018. Retrieved 16 March 2018.\n' + '
  490. \n' + '
  491. ^ de Vries, Alex (May 2018). "Bitcoin's Growing Energy Problem". Joule. 2 (5): 801–805. doi:10.1016/j.joule.2018.04.016.\n' + '
  492. \n' + '
  493. ^ a b Köhler, Susanne; Pizzol, Massimo (20 November 2019). "Life Cycle Assessment of Bitcoin Mining". Environmental Science & Technology. 53 (23): 13598–13606. Bibcode:2019EnST...5313598K. doi:10.1021/acs.est.9b05687. PMID 31746188.\n' + '
  494. \n' + '
  495. ^ Baraniuk, Chris (3 July 2019). "Bitcoin's global energy use 'equals Switzerland'". BBC News. Archived from the original on 16 January 2020. Retrieved 2 February 2020.\n' + '
  496. \n' + '
  497. ^ "How bad is Bitcoin for the environment really?". Independent. 12 February 2021. Retrieved 15 February 2021. requires nearly as much energy as the entire country of Argentina\n' + '
  498. \n' + '
  499. ^ O'Brien, Matt (13 June 2015). "The scam called Bitcoin". Daily Herald. Archived from the original on 16 June 2015. Retrieved 20 September 2016.\n' + '
  500. \n' + '
  501. ^ Potenza, Alessandra (21 December 2017). "Can renewable power offset bitcoin's massive energy demands?". TheVerge News. Archived from the original on 12 January 2018. Retrieved 12 January 2018.\n' + '
  502. \n' + '
  503. ^ Lampert, Allison (12 January 2018). "Chinese bitcoin miners eye sites in energy-rich Canada". Reuters. Archived from the original on 14 January 2018. Retrieved 14 January 2018.\n' + '
  504. \n' + '
  505. ^ "Bitcoin is literally ruining the earth, claim experts". The Independent. 6 December 2017. Archived from the original on 19 January 2018. Retrieved 23 January 2018.\n' + '
  506. \n' + '
  507. ^ "The Hard Math Behind Bitcoin's Global Warming Problem". WIRED. 15 December 2017. Archived from the original on 21 January 2018. Retrieved 23 January 2018.\n' + '
  508. \n' + '
  509. ^ a b Ponciano, Jonathan (18 April 2021). "Crypto Flash Crash Wiped Out $300 Billion In Less Than 24 Hours, Spurring Massive Bitcoin Liquidations". Forbes. Retrieved 24 April 2021.\n' + '
  510. \n' + '
  511. ^ Murtaugh, Dan (9 February 2021). "The Possible Xinjiang Coal Link in Tesla's Bitcoin Binge". Bloomberg.com. Retrieved 24 April 2021.\n' + '
  512. \n' + '
  513. ^ a b Tully, Shawn (20 April 2021). "Commentary: How much Bitcoin comes from dirty coal? A flooded mine in China just spotlighted the issue". Fortune. Retrieved 23 April 2021.\n' + '
  514. \n' + '
  515. ^ Chant, Tim De (10 May 2021). "Private-equity firm revives zombie fossil-fuel power plant to mine bitcoin". Ars Technica. Retrieved 10 May 2021.\n' + '
  516. \n' + '
  517. ^ Hern, Alex (17 January 2018). "Bitcoin's energy usage is huge – we can't afford to ignore it". The Guardian. Archived from the original on 23 January 2018. Retrieved 18 September 2019.\n' + '
  518. \n' + '
  519. ^ Ethan, Lou (17 January 2019). "Bitcoin as big oil: the next big environmental fight?". The Guardian. Archived from the original on 29 August 2019. Retrieved 18 September 2019.\n' + '
  520. \n' + '
  521. ^ Foteinis, Spyros (2018). "Bitcoin's alarming carbon footprint". Nature. 554 (7691): 169. Bibcode:2018Natur.554..169F. doi:10.1038/d41586-018-01625-x.\n' + '
  522. \n' + '
  523. ^ Krause, Max J.; Tolaymat, Thabet (2018). "Quantification of energy and carbon costs for mining cryptocurrencies". Nature Sustainability. 1 (11): 711–718. doi:10.1038/s41893-018-0152-7. S2CID 169170289.\n' + '
  524. \n' + '
  525. ^ a b Mora, Camilo; et al. (2018). "Bitcoin emissions alone could push global warming above 2°C". Nature Climate Change. 8 (11): 931–933. Bibcode:2018NatCC...8..931M. doi:10.1038/s41558-018-0321-8. S2CID 91491182.\n' + '
  526. \n' + '
  527. ^ a b Stoll, Christian; Klaaßen, Lena; Gallersdörfer, Ulrich (2019). "The Carbon Footprint of Bitcoin". Joule. 3 (7): 1647–1661. doi:10.1016/j.joule.2019.05.012.\n' + '
  528. \n' + '
  529. ^ Masanet, Eric; et al. (2019). "Implausible projections overestimate near-term Bitcoin CO
    2
    emissions". Nature Climate Change. 9 (9): 653–654. Bibcode:2019NatCC...9..653M. doi:10.1038/s41558-019-0535-4. hdl:2066/207817. OSTI 1561950.
    \n' + '
  530. \n' + '
  531. ^ Dittmar, Lars; Praktiknjo, Aaron (2019). "Could Bitcoin emissions push global warming above 2°C?". Nature Climate Change. 9 (9): 656–657. Bibcode:2019NatCC...9..656D. doi:10.1038/s41558-019-0534-5.\n' + '
  532. \n' + '
  533. ^ Houy, Nicolas (2019). "Rational mining limits Bitcoin emissions". Nature Climate Change. 9 (9): 655. Bibcode:2019NatCC...9..655H. doi:10.1038/s41558-019-0533-6.\n' + '
  534. \n' + '
  535. ^ Kamiya, George. "Commentary: Bitcoin energy use - mined the gap". iea.org. Archived from the original on 3 March 2020. Retrieved 5 December 2019.\n' + '
  536. \n' + '
  537. ^ Lavin, Tim (8 August 2013). "The SEC Shows Why Bitcoin Is Doomed". bloomberg.com. Bloomberg LP. Archived from the original on 25 March 2014. Retrieved 20 October 2013.\n' + '
  538. \n' + '
  539. ^ Lee, Timothy B. (21 November 2013). "Here's how Bitcoin charmed Washington". The Washington Post. Archived from the original on 1 January 2017. Retrieved 10 October 2016.\n' + '
  540. \n' + '
  541. ^ Popper, Nathaniel (13 July 2018). "How Russian Spies Hid Behind Bitcoin in Hacking Campaign". NYT. Archived from the original on 14 July 2018. Retrieved 14 July 2018.\n' + '
  542. \n' + '
  543. ^ Ehrlich, Steven. "Janet Yellen, Bitcoin And Crypto Fearmongers Get Pushback From Former CIA Director". Forbes.\n' + '
  544. \n' + '
  545. ^ Ball, James (22 March 2013). "Silk Road: the online drug marketplace that officials seem powerless to stop". theguardian.com. Guardian News and Media Limited. Archived from the original on 12 October 2013. Retrieved 20 October 2013.\n' + '
  546. \n' + '
  547. ^ Montag, Ali (9 July 2018). "Nobel-winning economist: Authorities will bring down 'hammer' on bitcoin". CNBC. Archived from the original on 11 July 2018. Retrieved 11 July 2018.\n' + '
  548. \n' + '
  549. ^ Newlands, Chris (9 July 2018). "Stiglitz, Roubini and Rogoff lead joint attack on bitcoin". Financial News. Archived from the original on 11 July 2018. Retrieved 11 July 2018.\n' + '
  550. \n' + '
  551. ^ Foley, Sean; Karlsen, Jonathan R.; Putniņš, Tālis J. (19 February 2018). "Sex, drugs, and bitcoin: How much illegal activity is financed through cryptocurrencies?". University of Oxford Faculty of Law. Oxford Business Law Blog. Archived from the original on 10 June 2018. Retrieved 11 June 2018.\n' + '
  552. \n' + '
  553. ^ Foley, Sean; Karlsen, Jonathan R.; Putniņš, Tālis J. (30 January 2018). "Sex, Drugs, and Bitcoin: How Much Illegal Activity Is Financed Through Cryptocurrencies?". Social Science Research Network. SSRN 3102645.\n' + '
  554. \n' + '
  555. ^ Antonopoulos, Andreas (2017). "3". Mastering Bitcoin: Programming the Open Blockchain (2nd ed.). O'Reilly Media. ISBN 978-1491954386. Bitcoin Core is the reference implementation of the bitcoin system, meaning that it is the authoritative reference on how each part of the technology should be implemented. Bitcoin Core implements all aspects of bitcoin, including wallets, a transaction and block validation engine, and a full network node in the peer-to-peer bitcoin network.\n' + '
  556. \n' + '
  557. ^ "Bitcoin Core version 0.9.0 released". Bitcoin Core. 19 March 2014. Retrieved 21 October 2018.\n' + '
  558. \n' + '
  559. ^ a b c Antonopoulos, Andreas M. (2014). Mastering Bitcoin: Unlocking Digital Cryptocurrencies. O'Reilly Media, Inc. pp. 31–32. ISBN 978-1491902646. Retrieved 6 November 2016.\n' + '
  560. \n' + '
  561. ^ "MIT Announces $900,000 Bitcoin Developer Fund". Inc. 29 March 2016. Retrieved 21 October 2018.\n' + '
  562. \n' + '
  563. ^ a b "About". Bitcoin Core. Retrieved 21 October 2018.\n' + '
  564. \n' + '
  565. ^ "Bitcoin Developer Examples". Bitcoin. Retrieved 21 October 2018.\n' + '
  566. \n' + '
  567. ^ "checkpoints.cpp". Repository source code. GitHub, Inc. Retrieved 13 November 2016.\n' + '
  568. \n' + '
  569. ^ "bitcoin/chainparams.cpp". GitHub. Retrieved 21 October 2018.\n' + '
  570. \n' + '
  571. ^ Mike Orcutt (19 May 2015). "Leaderless Bitcoin Struggles to Make Its Most Crucial Decision". MIT Technology Review. Retrieved 15 November 2016.\n' + '
  572. \n' + '
  573. ^ "Alert System Retirement". Bitcoin Project. 1 November 2016. Retrieved 16 November 2016.\n' + '
  574. \n' + '
  575. ^ Antonopoulos, Andreas (29 May 2013). "Bitcoin is a money platform with many APIs". Radar. O'Reilly. Retrieved 19 November 2016.\n' + '
  576. \n' + '
  577. ^ "Bitcoin Core devtools README - Create and verify timestamps of merge commits". GitHub. Retrieved 5 May 2018.\n' + '
  578. \n' + '
  579. ^ a b Preukschat, Alex; Josep Busquet (2015). Bitcoin: The Hunt of Satoshi Nakamoto. Europe Comics. p. 87. ISBN 9791032800201. Retrieved 16 November 2016.\n' + '
  580. \n' + '
  581. ^ Maria Bustillos (25 August 2015). "Inside the Fight Over Bitcoin's Future". New Yorker. Retrieved 29 June 2020.\n' + '
  582. \n' + '
  583. ^ Shin, Laura. "Bitcoin And Taxes: If Not HODLing, Consider Donating". Forbes. Archived from the original on 22 December 2017. Retrieved 22 December 2017.\n' + '
  584. \n' + '
  585. ^ Kaminska, Izabella (22 December 2017). "The HODL". Financial Times. Retrieved 21 November 2018.\n' + '
  586. \n' + '
  587. ^ Montag, Ali (26 August 2018). "'HODL,' 'whale' and 5 other cryptocurrency slang terms explained". CNBC. Retrieved 12 November 2020.\n' + '
  588. \n' + '
  589. ^ Wong, Joon Ian. "Buy and hodl, just don't get #rekt: The slang that gets you taken seriously as a bitcoin trader". Quartz. Archived from the original on 22 December 2017. Retrieved 22 December 2017.\n' + '
  590. \n' + '
  591. ^ Akhtar, Tanzeel (22 December 2017). "As Bitcoin plunges, cryptocurrency fans chant 'HODL' for comfort". TheStreet. Archived from the original on 23 December 2017. Retrieved 22 December 2017.\n' + '
  592. \n' + '
  593. ^ Hajric, Vildana (19 November 2020). "All the Bitcoin Lingo You Need to Know as Crypto Heats Up". Bloomberg. Retrieved 1 December 2020.\n' + '
  594. \n' + '
  595. ^ Stross, Charles (2013). Neptune's Brood (First ed.). New York: Penguin Group USA. ISBN 978-0-425-25677-0. It's theft-proof too – for each bitcoin is cryptographically signed by the mind of its owner.\n' + '
  596. \n' + '
  597. ^ "Crib Sheet: Neptune's Brood – Charlie's Diary". www.antipope.org. Archived from the original on 14 June 2017. Retrieved 5 December 2017. I wrote Neptune's Brood in 2011. Bitcoin was obscure back then, and I figured had just enough name recognition to be a useful term for an interstellar currency: it'd clue people in that it was a networked digital currency.\n' + '
  598. \n' + '
  599. ^ Kenigsberg, Ben (2 October 2014). "Financial Wild West". The New York Times. Archived from the original on 18 May 2015. Retrieved 8 May 2015.\n' + '
  600. \n' + '
  601. ^ Michel, Lincoln (16 December 2017). "What the Hell Is Bitcoin? Let This Documentary on Netflix Explain". GQ. Archived from the original on 18 November 2018. Retrieved 10 October 2018.\n' + '
  602. \n' + '
  603. ^ "Introducing Ledger, the First Bitcoin-Only Academic Journal". Motherboard. Archived from the original on 10 January 2017.\n' + '
  604. \n' + '
  605. ^ "Editorial Policies". ledgerjournal.org. Archived from the original on 23 December 2016. Retrieved 10 January 2017.\n' + '
  606. \n' + '
  607. ^ "How to Write and Format an Article for Ledger" (PDF). Ledger. 2015. doi:10.5195/LEDGER.2015.1 (inactive 19 January 2021). Archived (PDF) from the original on 22 September 2015.CS1 maint: DOI inactive as of January 2021 (link)\n' + '
  608. \n' + '
\n' + '

External links

\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '
\n' + '

Page 2

\n' + '
Optical machine-readable representation of data
\n' + '\n' + '
A UPC-A barcode
\n' + '

A barcode or bar code is a method of representing data in a visual, machine-readable form. Initially, barcodes represented data by varying the widths and spacings of parallel lines. These barcodes, now commonly referred to as linear or one-dimensional (1D), can be scanned by special optical scanners, called barcode readers, of which there are several types. Later, two-dimensional (2D) variants were developed, using rectangles, dots, hexagons and other patterns, called matrix codes or 2D barcodes, although they do not use bars as such. 2D barcodes can be read using purpose-built 2D optical scanners, which exist in a few different forms. 2D barcodes can also be read by a digital camera connected to a microcomputer running software that takes a photographic image of the barcode and analyzes the image to deconstruct and decode the 2D barcode. A mobile device with an inbuilt camera, such as smartphone, can function as the latter type of 2D barcode reader using specialized application software. (The same sort of mobile device could also read 1D barcodes, depending on the application software.)\n' + '

The barcode was invented by Norman Joseph Woodland and Bernard Silver and patented in the US in 1951.[1] The invention was based on Morse code[2] that was extended to thin and thick bars. However, it took over twenty years before this invention became commercially successful. An early use of one type of barcode in an industrial context was sponsored by the Association of American Railroads in the late 1960s. Developed by General Telephone and Electronics (GTE) and called KarTrak ACI (Automatic Car Identification), this scheme involved placing colored stripes in various combinations on steel plates which were affixed to the sides of railroad rolling stock. Two plates were used per car, one on each side, with the arrangement of the colored stripes encoding information such as ownership, type of equipment, and identification number.[3] The plates were read by a trackside scanner, located for instance, at the entrance to a classification yard, while the car was moving past.[4] The project was abandoned after about ten years because the system proved unreliable after long-term use.[3]\n' + '

Barcodes became commercially successful when they were used to automate supermarket checkout systems, a task for which they have become almost universal. The Uniform Grocery Product Code Council had chosen, in 1973, the barcode design developed by George Laurer. Laurer's barcode, with vertical bars, printed better than the circular barcode developed by Woodland and Silver.[5] Their use has spread to many other tasks that are generically referred to as automatic identification and data capture (AIDC). The first scanning of the now-ubiquitous Universal Product Code (UPC) barcode was on a pack of Wrigley Company chewing gum in June 1974 at a Marsh supermarket in Troy, Ohio, using scanner produced by Photographic Sciences Corporation.[6][5] QR codes, a specific type of 2D barcode, have recently become very popular due to the growth in smartphone ownership.[7]\n' + '

Other systems have made inroads in the AIDC market, but the simplicity, universality and low cost of barcodes has limited the role of these other systems, particularly before technologies such as radio-frequency identification (RFID) became available after 1995.\n' + '

\n' + '\n' + '\n' + '

History

\n' + '\n' + '

In 1948 Bernard Silver, a graduate student at Drexel Institute of Technology in Philadelphia, Pennsylvania, US overheard the president of the local food chain, Food Fair, asking one of the deans to research a system to automatically read product information during checkout.[8] Silver told his friend Norman Joseph Woodland about the request, and they started working on a variety of systems. Their first working system used ultraviolet ink, but the ink faded too easily and was expensive.[9]\n' + '

Convinced that the system was workable with further development, Woodland left Drexel, moved into his father's apartment in Florida, and continued working on the system. His next inspiration came from Morse code, and he formed his first barcode from sand on the beach. "I just extended the dots and dashes downwards and made narrow lines and wide lines out of them."[9] To read them, he adapted technology from optical soundtracks in movies, using a 500-watt incandescent light bulb shining through the paper onto an RCA935 photomultiplier tube (from a movie projector) on the far side. He later decided that the system would work better if it were printed as a circle instead of a line, allowing it to be scanned in any direction.\n' + '

On 20 October 1949, Woodland and Silver filed a patent application for "Classifying Apparatus and Method", in which they described both the linear and bull's eye printing patterns, as well as the mechanical and electronic systems needed to read the code. The patent was issued on 7 October 1952 as US Patent 2,612,994.[1] In 1951, Woodland moved to IBM and continually tried to interest IBM in developing the system. The company eventually commissioned a report on the idea, which concluded that it was both feasible and interesting, but that processing the resulting information would require equipment that was some time off in the future.\n' + '

IBM offered to buy the patent, but the offer was not accepted. Philco purchased the patent in 1962 and then sold it to RCA sometime later.[9]\n' + '

\n' + '

Collins at Sylvania

\n' + '

During his time as an undergraduate, David Jarrett Collins worked at the Pennsylvania Railroad and became aware of the need to automatically identify railroad cars. Immediately after receiving his master's degree from MIT in 1959, he started work at GTE Sylvania and began addressing the problem. He developed a system called KarTrak using blue and red reflective stripes attached to the side of the cars, encoding a six-digit company identifier and a four-digit car number.[9] Light reflected off the colored stripes was read by photomultiplier vacuum tubes.[10]\n' + '

The Boston and Maine Railroad tested the KarTrak system on their gravel cars in 1961. The tests continued until 1967, when the Association of American Railroads (AAR) selected it as a standard, Automatic Car Identification, across the entire North American fleet. The installations began on 10 October 1967. However, the economic downturn and rash of bankruptcies in the industry in the early 1970s greatly slowed the rollout, and it was not until 1974 that 95% of the fleet was labeled. To add to its woes, the system was found to be easily fooled by dirt in certain applications, which greatly affected accuracy. The AAR abandoned the system in the late 1970s, and it was not until the mid-1980s that they introduced a similar system, this time based on radio tags.[11]\n' + '

The railway project had failed, but a toll bridge in New Jersey requested a similar system so that it could quickly scan for cars that had purchased a monthly pass. Then the U.S. Post Office requested a system to track trucks entering and leaving their facilities. These applications required special retroreflector labels. Finally, Kal Kan asked the Sylvania team for a simpler (and cheaper) version which they could put on cases of pet food for inventory control.\n' + '

\n' + '

Computer Identics Corporation

\n' + '

In 1967, with the railway system maturing, Collins went to management looking for funding for a project to develop a black-and-white version of the code for other industries. They declined, saying that the railway project was large enough, and they saw no need to branch out so quickly.\n' + '

Collins then quit Sylvania and formed the Computer Identics Corporation.[9] As its first innovations, Computer Identics moved from using incandescent light bulbs in its systems, replacing them with helium–neon lasers, and incorporated a mirror as well, making it capable of locating a barcode up to several feet in front of the scanner. This made the entire process much simpler and more reliable, and typically enabled these devices to deal with damaged labels, as well, by recognizing and reading the intact portions.\n' + '

Computer Identics Corporation installed one of its first two scanning systems in the spring of 1969 at a General Motors (Buick) factory in Flint, Michigan.[9] The system was used to identify a dozen types of transmissions moving on an overhead conveyor from production to shipping. The other scanning system was installed at General Trading Company's distribution center in Carlstadt, New Jersey to direct shipments to the proper loading bay.\n' + '

\n' + '

Universal Product Code

\n' + '\n' + '

In 1966, the National Association of Food Chains (NAFC) held a meeting on the idea of automated checkout systems. RCA, who had purchased the rights to the original Woodland patent, attended the meeting and initiated an internal project to develop a system based on the bullseye code. The Kroger grocery chain volunteered to test it.\n' + '

In the mid-1970s, the NAFC established the Ad-Hoc Committee for U.S. Supermarkets on a Uniform Grocery-Product Code to set guidelines for barcode development. In addition, it created a symbol-selection subcommittee to help standardize the approach. In cooperation with consulting firm, McKinsey & Co., they developed a standardized 11-digit code for identifying products. The committee then sent out a contract tender to develop a barcode system to print and read the code. The request went to Singer, National Cash Register (NCR), Litton Industries, RCA, Pitney-Bowes, IBM and many others.[12] A wide variety of barcode approaches was studied, including linear codes, RCA's bullseye concentric circle code, starburst patterns and others.\n' + '

In the spring of 1971, RCA demonstrated their bullseye code at another industry meeting. IBM executives at the meeting noticed the crowds at the RCA booth and immediately developed their own system. IBM marketing specialist Alec Jablonover remembered that the company still employed Woodland, and he[who?] established a new facility in Raleigh-Durham Research Triangle Park to lead development.\n' + '

In July 1972, RCA began an 18-month test in a Kroger store in Cincinnati. Barcodes were printed on small pieces of adhesive paper, and attached by hand by store employees when they were adding price tags. The code proved to have a serious problem; the printers would sometimes smear ink, rendering the code unreadable in most orientations. However, a linear code, like the one being developed by Woodland at IBM, was printed in the direction of the stripes, so extra ink would simply make the code "taller" while remaining readable. So on 3 April 1973, the IBM UPC was selected as the NAFC standard. IBM had designed five versions of UPC symbology for future industry requirements: UPC A, B, C, D, and E.[13]\n' + '

NCR installed a testbed system at Marsh's Supermarket in Troy, Ohio, near the factory that was producing the equipment. On 26 June 1974, Clyde Dawson pulled a 10-pack of Wrigley's Juicy Fruit gum out of his basket and it was scanned by Sharon Buchanan at 8:01 am. The pack of gum and the receipt are now on display in the Smithsonian Institution. It was the first commercial appearance of the UPC.[14]\n' + '

In 1971, an IBM team was assembled for an intensive planning session, threshing out, 12 to 18 hours a day, how the technology would be deployed and operate cohesively across the system, and scheduling a roll-out plan. By 1973, the team were meeting with grocery manufacturers to introduce the symbol that would need to be printed on the packaging or labels of all of their products. There were no cost savings for a grocery to use it, unless at least 70% of the grocery's products had the barcode printed on the product by the manufacturer. IBM projected that 75% would be needed in 1975. Yet, although this was achieved, there were still scanning machines in fewer than 200 grocery stores by 1977.[15]\n' + '

Economic studies conducted for the grocery industry committee projected over $40 million in savings to the industry from scanning by the mid-1970s. Those numbers were not achieved in that time-frame and some predicted the demise of barcode scanning. The usefulness of the barcode required the adoption of expensive scanners by a critical mass of retailers while manufacturers simultaneously adopted barcode labels. Neither wanted to move first and results were not promising for the first couple of years, with Business Week proclaiming "The Supermarket Scanner That Failed" in a 1976 article.[14][16]\n' + '

On the other hand, experience with barcode scanning in those stores revealed additional benefits. The detailed sales information acquired by the new systems allowed greater responsiveness to customer habits, needs and preferences. This was reflected in the fact that about 5 weeks after installing barcode scanners, sales in grocery stores typically started climbing and eventually leveled off at a 10–12% increase in sales that never dropped off. There was also a 1–2% decrease in operating cost for those stores, and this enabled them to lower prices and thereby to increase market share. It was shown in the field that the return on investment for a barcode scanner was 41.5%. By 1980, 8,000 stores per year were converting.[15]\n' + '

Sims Supermarkets were the first location in Australia to use barcodes, starting in 1979.[17]\n' + '

\n' + '

Industrial adoption

\n' + '

In 1981, the United States Department of Defense adopted the use of Code 39 for marking all products sold to the United States military. This system, Logistics Applications of Automated Marking and Reading Symbols (LOGMARS), is still used by DoD and is widely viewed as the catalyst for widespread adoption of barcoding in industrial uses.[18]\n' + '

\n' + '

Use

\n' + '\n' + '
Barcode on a patient identification wristband
\n' + '
Barcoded parcel
\n' + '

Barcodes are widely used around the world in many contexts. In stores, UPC barcodes are pre-printed on most items other than fresh produce from a grocery store. This speeds up processing at check-outs and helps track items and also reduces instances of shoplifting involving price tag swapping, although shoplifters can now print their own barcodes.[19] Barcodes that encode a book's ISBN are also widely pre-printed on books, journals and other printed materials. In addition, retail chain membership cards use barcodes to identify customers, allowing for customized marketing and greater understanding of individual consumer shopping patterns. At the point of sale, shoppers can get product discounts or special marketing offers through the address or e-mail address provided at registration.\n' + '

Barcodes are widely used in the healthcare and hospital settings, ranging from patient identification (to access patient data, including medical history, drug allergies, etc.) to creating SOAP Notes[20] with barcodes to medication management. They are also used to facilitate the separation and indexing of documents that have been imaged in batch scanning applications, track the organization of species in biology,[21] and integrate with in-motion checkweighers to identify the item being weighed in a conveyor line for data collection.\n' + '

They can also be used to keep track of objects and people; they are used to keep track of rental cars, airline luggage, nuclear waste, registered mail, express mail and parcels. Barcoded tickets (which may be printed by the customer on their home printer, or stored on their mobile device) allow the holder to enter sports arenas, cinemas, theatres, fairgrounds, and transportation, and are used to record the arrival and departure of vehicles from rental facilities etc. This can allow proprietors to identify duplicate or fraudulent tickets more easily. Barcodes are widely used in shop floor control applications software where employees can scan work orders and track the time spent on a job.\n' + '

Barcodes are also used in some kinds of non-contact 1D and 2D position sensors. A series of barcodes are used in some kinds of absolute 1D linear encoder. The barcodes are packed close enough together that the reader always has one or two barcodes in its field of view. As a kind of fiducial marker, the relative position of the barcode in the field of view of the reader gives incremental precise positioning, in some cases with sub-pixel resolution. The data decoded from the barcode gives the absolute coarse position. An "address carpet", such as Howell's binary pattern and the Anoto dot pattern, is a 2D barcode designed so that a reader, even though only a tiny portion of the complete carpet is in the field of view of the reader, can find its absolute X,Y position and rotation in the carpet.[22][23]\n' + '

2D barcodes can embed a hyperlink to a web page. A mobile device with an inbuilt camera might be used to read the pattern and browse the linked website, which can help a shopper find the best price for an item in the vicinity. Since 2005, airlines use an IATA-standard 2D barcode on boarding passes (Bar Coded Boarding Pass (BCBP)), and since 2008 2D barcodes sent to mobile phones enable electronic boarding passes.[24]\n' + '

Some applications for barcodes have fallen out of use. In the 1970s and 1980s, software source code was occasionally encoded in a barcode and printed on paper (Cauzin Softstrip and Paperbyte[25] are barcode symbologies specifically designed for this application), and the 1991 Barcode Battler computer game system used any standard barcode to generate combat statistics.\n' + '

Artists have used barcodes in art, such as Scott Blake's Barcode Jesus, as part of the post-modernism movement.\n' + '

\n' + '

Symbologies

\n' + '

The mapping between messages and barcodes is called a symbology. The specification of a symbology includes the encoding of the message into bars and spaces, any required start and stop markers, the size of the quiet zone required to be before and after the barcode, and the computation of a checksum.\n' + '

Linear symbologies can be classified mainly by two properties:\n' + '

\n' + '
Continuous vs. discrete
\n' + '
  • Characters in discrete symbologies are composed of n bars and n − 1 spaces. There is an additional space between characters, but it does not convey information, and may have any width as long as it is not confused with the end of the code.
  • \n' + '
  • Characters in continuous symbologies are composed of n bars and n spaces, and usually abut, with one character ending with a space and the next beginning with a bar, or vice versa. A special end pattern that has bars on both ends is required to end the code.
\n' + '
Two-width vs. many-width
\n' + '
  • A two-width, also called a binary bar code, contains bars and spaces of two widths, "wide" and "narrow". The precise width of the wide bars and spaces is not critical; typically it is permitted to be anywhere between 2 and 3 times the width of the narrow equivalents.
  • \n' + '
  • Some other symbologies use bars of two different heights (POSTNET), or the presence or absence of bars (CPC Binary Barcode). These are normally also considered binary bar codes.
  • \n' + '
  • Bars and spaces in many-width symbologies are all multiples of a basic width called the module; most such codes use four widths of 1, 2, 3 and 4 modules.
\n' + '

Some symbologies use interleaving. The first character is encoded using black bars of varying width. The second character is then encoded by varying the width of the white spaces between these bars. Thus characters are encoded in pairs over the same section of the barcode. Interleaved 2 of 5 is an example of this.\n' + '

Stacked symbologies repeat a given linear symbology vertically.\n' + '

The most common among the many 2D symbologies are matrix codes, which feature square or dot-shaped modules arranged on a grid pattern. 2D symbologies also come in circular and other patterns and may employ steganography, hiding modules within an image (for example, DataGlyphs).\n' + '

Linear symbologies are optimized for laser scanners, which sweep a light beam across the barcode in a straight line, reading a slice of the barcode light-dark patterns. Scanning at an angle makes the modules appear wider, but does not change the width ratios. Stacked symbologies are also optimized for laser scanning, with the laser making multiple passes across the barcode.\n' + '

In the 1990s development of charge-coupled device (CCD) imagers to read barcodes was pioneered by Welch Allyn. Imaging does not require moving parts, as a laser scanner does. In 2007, linear imaging had begun to supplant laser scanning as the preferred scan engine for its performance and durability.\n' + '

2D symbologies cannot be read by a laser, as there is typically no sweep pattern that can encompass the entire symbol. They must be scanned by an image-based scanner employing a CCD or other digital camera sensor technology.\n' + '

\n' + '

Barcode readers

\n' + '\n' + '
GTIN barcodes on Coke bottles. The images at right show how the laser of barcode readers "see" the images behind a red filter.
\n' + '

The earliest, and still the cheapest, barcode scanners are built from a fixed light and a single photosensor that is manually moved across the barcode. Barcode scanners can be classified into three categories based on their connection to the computer. The older type is the RS-232 barcode scanner. This type requires special programming for transferring the input data to the application program. Keyboard interface scanners connect to a computer using a PS/2 or AT keyboard–compatible adaptor cable (a "keyboard wedge"). The barcode's data is sent to the computer as if it had been typed on the keyboard.\n' + '

Like the keyboard interface scanner, USB scanners do not need custom code for transferring input data to the application program. On PCs running Windows the human interface device emulates the data merging action of a hardware "keyboard wedge", and the scanner automatically behaves like an additional keyboard.\n' + '

Most modern smartphones are able to decode barcode using their built-in camera. Google's mobile Android operating system can use their own Google Lens application to scan QR codes, or third party apps like Barcode Scanner to read both one-dimensional barcodes and QR codes. Nokia's Symbian operating system featured a barcode scanner,[26] while mbarcode[27] is a QR code reader for the Maemo operating system. In Apple iOS 11, the native camera app can decode QR codes and can link to URLs, join wireless networks, or perform other operations depending on the QR Code contents.[28] Other paid and free apps are available with scanning capabilities for other symbologies or for earlier iOS versions.[29] With BlackBerry devices, the App World application can natively scan barcodes and load any recognized Web URLs on the device's Web browser. Windows Phone 7.5 is able to scan barcodes through the Bing search app. However, these devices are not designed specifically for the capturing of barcodes. As a result, they do not decode nearly as quickly or accurately as a dedicated barcode scanner or portable data terminal.[citation needed]\n' + '

\n' + '

Quality control and verification

\n' + '

It is common for producers and users of bar codes to have a quality management system which includes verification and validation of bar codes.[30] Barcode verification examines scanability and the quality of the barcode in comparison to industry standards and specifications.[31] Barcode verifiers are primarily used by businesses that print and use barcodes. Any trading partner in the supply chain can test barcode quality. It is important to verify a barcode to ensure that any reader in the supply chain can successfully interpret a barcode with a low error rate. Retailers levy large penalties for non-compliant barcodes. These chargebacks can reduce a manufacturer's revenue by 2% to 10%.[32]\n' + '

A barcode verifier works the way a reader does, but instead of simply decoding a barcode, a verifier performs a series of tests. For linear barcodes these tests are:\n' + '

\n' + '
  • Edge contrast (EC)[33]\n' + '
    • The difference between the space reflectance (Rs) and adjoining bar reflectance (Rb). EC=Rs-Rb
  • \n' + '
  • Minimum bar reflectance (Rb)[33]\n' + '
    • The smallest reflectance value in a bar.
  • \n' + '
  • Minimum space reflectance (Rs)[33]\n' + '
    • The smallest reflectance value in a space.
  • \n' + '
  • Symbol contrast (SC)[33]\n' + '
    • Symbol Contrast is the difference in reflectance values of the lightest space (including the quiet zone) and the darkest bar of the symbol. The greater the difference, the higher the grade. The parameter is graded as either A, B, C, D, or F. SC=Rmax-Rmin
  • \n' + '
  • Minimum edge contrast (ECmin)[33]\n' + '
    • The difference between the space reflectance (Rs) and adjoining bar reflectance (Rb). EC=Rs-Rb
  • \n' + '
  • Modulation (MOD)[33]\n' + '
    • The parameter is graded either A, B, C, D, or F. This grade is based on the relationship between minimum edge contrast (ECmin) and symbol contrast (SC). MOD=ECmin/SC The greater the difference between minimum edge contrast and symbol contrast, the lower the grade. Scanners and verifiers perceive the narrower bars and spaces to have less intensity than wider bars and spaces; the comparison of the lesser intensity of narrow elements to the wide elements is called modulation. This condition is affected by aperture size.
  • \n' + '
  • Inter-character gap[33]\n' + '
    • In discrete barcodes, the space that disconnects the two contiguous characters. When present, inter-character gaps are considered spaces (elements) for purposes of edge determination and reflectance parameter grades.
  • \n' + '
  • Defects
  • \n' + '
  • Decode[33]\n' + '
    • Extracting the information which has been encoded in a bar code symbol.
  • \n' + '
  • Decodability[33]\n' + '
    • Can be graded as A, B, C, D, or F. The Decodability grade indicates the amount of error in the width of the most deviant element in the symbol. The less deviation in the symbology, the higher the grade. Decodability is a measure of print accuracy using the symbology reference decode algorithm.
\n' + '

2D matrix symbols look at the parameters:\n' + '

\n' + '
  • Symbol contrast[33]
  • \n' + '
  • Modulation[33]
  • \n' + '
  • Decode[33]
  • \n' + '
  • Unused error correction
  • \n' + '
  • Fixed (finder) pattern damage
  • \n' + '
  • Grid non-uniformity
  • \n' + '
  • Axial non-uniformity[34]
\n' + '

Depending on the parameter, each ANSI test is graded from 0.0 to 4.0 (F to A), or given a pass or fail mark. Each grade is determined by analyzing the scan reflectance profile (SRP), an analog graph of a single scan line across the entire symbol. The lowest of the 8 grades is the scan grade, and the overall ISO symbol grade is the average of the individual scan grades. For most applications a 2.5 (C) is the minimal acceptable symbol grade.[35]\n' + '

Compared with a reader, a verifier measures a barcode's optical characteristics to international and industry standards. The measurement must be repeatable and consistent. Doing so requires constant conditions such as distance, illumination angle, sensor angle and verifier aperture. Based on the verification results, the production process can be adjusted to print higher quality barcodes that will scan down the supply chain.\n' + '

Bar code validation may include evaluations after use (and abuse) testing such as sunlight, abrasion, impact, moisture, etc.[36]\n' + '

\n' + '

Barcode verifier standards

\n' + '

Barcode verifier standards are defined by the International Organization for Standardization (ISO), in ISO/IEC 15426-1 (linear) or ISO/IEC 15426-2 (2D).[citation needed] The current international barcode quality specification is ISO/IEC 15416 (linear) and ISO/IEC 15415 (2D).[citation needed] The European Standard EN 1635 has been withdrawn and replaced by ISO/IEC 15416. The original U.S. barcode quality specification was ANSI X3.182. (UPCs used in the US – ANSI/UCC5).[citation needed] As of 2011 the ISO workgroup JTC1 SC31 was developing a Direct Part Marking (DPM) quality standard: ISO/IEC TR 29158.[37]\n' + '

\n' + '

Benefits

\n' + '

In point-of-sale management, barcode systems can provide detailed up-to-date information on the business, accelerating decisions and with more confidence. For example:\n' + '

\n' + '
  • Fast-selling items can be identified quickly and automatically reordered.
  • \n' + '
  • Slow-selling items can be identified, preventing inventory build-up.
  • \n' + '
  • The effects of merchandising changes can be monitored, allowing fast-moving, more profitable items to occupy the best space.
  • \n' + '
  • Historical data can be used to predict seasonal fluctuations very accurately.
  • \n' + '
  • Items may be repriced on the shelf to reflect both sale prices and price increases.
  • \n' + '
  • This technology also enables the profiling of individual consumers, typically through a voluntary registration of discount cards. While pitched as a benefit to the consumer, this practice is considered to be potentially dangerous by privacy advocates.[which?]
\n' + '

Besides sales and inventory tracking, barcodes are very useful in logistics and supply chain management.\n' + '

\n' + '
  • When a manufacturer packs a box for shipment, a Unique Identifying Number (UID) can be assigned to the box.
  • \n' + '
  • A database can link the UID to relevant information about the box; such as order number, items packed, quantity packed, destination, etc.
  • \n' + '
  • The information can be transmitted through a communication system such as Electronic Data Interchange (EDI) so the retailer has the information about a shipment before it arrives.
  • \n' + '
  • Shipments that are sent to a Distribution Center (DC) are tracked before forwarding. When the shipment reaches its final destination, the UID gets scanned, so the store knows the shipment's source, contents, and cost.
\n' + '

Barcode scanners are relatively low cost and extremely accurate compared to key-entry, with only about 1 substitution error in 15,000 to 36 trillion characters entered.[38][unreliable source?] The exact error rate depends on the type of barcode.\n' + '

\n' + '

Types of barcodes

\n' + '

Linear barcodes

\n' + '

A first generation, "one dimensional" barcode that is made up of lines and spaces of various widths that create specific patterns.\n' + '

\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '
ExampleSymbologyContinuous or discreteBar widthsUses\n' + '
Australia Post 4-state barcode.pngAustralia Post barcodeDiscrete4 bar heightsAn Australia Post barcode as used on a business reply paid envelope and applied by automated sorting machines to other mail when initially processed in fluorescent ink .\n' + '
Codabar.svgCodabarDiscreteTwoOld format used in libraries and blood banks and on airbills (out of date, but still widely used in libraries)\n' + '
Code 25 – Non-interleaved 2 of 5ContinuousTwoIndustrial\n' + '
Barcode2of5example.svgCode 25 – Interleaved 2 of 5ContinuousTwoWholesale, libraries International standard ISO/IEC 16390\n' + '
Code11 barcode.pngCode 11DiscreteTwoTelephones (out of date)\n' + '
Code32 01234567.pngFarmacode or Code 32DiscreteTwoItalian pharmacode – use Code 39 (no international standard available)\n' + '
Code 3 of 9.svgCode 39DiscreteTwoVarious – international standard ISO/IEC 16388\n' + '
Code 49 wikipedia.pngCode 49ContinuousManyVarious\n' + '
Code 93 wikipedia.pngCode 93ContinuousManyVarious\n' + '
Code 128B-2009-06-02.svgCode 128ContinuousManyVarious – International Standard ISO/IEC 15417\n' + '
CPC BinaryDiscreteTwo\n' + '
Dx-film-edge-barcode.jpgDX film edge barcodeNeitherTall/shortColor print film\n' + '
Issn barcode.pngEAN 2ContinuousManyAddon code (magazines), GS1-approved – not an own symbology – to be used only with an EAN/UPC according to ISO/IEC 15420\n' + '
Isbn add5.pngEAN 5ContinuousManyAddon code (books), GS1-approved – not an own symbology – to be used only with an EAN/UPC according to ISO/IEC 15420\n' + '
EAN8.svgEAN-8, EAN-13ContinuousManyWorldwide retail, GS1-approved – International Standard ISO/IEC 15420\n' + '
Facing Identification MarkDiscreteTwoUSPS business reply mail\n' + '
Gs1-128 example.svgGS1-128 (formerly named UCC/EAN-128), incorrectly referenced as EAN 128 and UCC 128ContinuousManyVarious, GS1-approved – just an application of the Code 128 (ISO/IEC 15417) using the ANS MH10.8.2 AI Datastructures. It is not a separate symbology.\n' + '
Databar 14 00075678164125.pngGS1 DataBar, formerly Reduced Space Symbology (RSS)ContinuousManyVarious, GS1-approved\n' + '
Intelligent Mail Barcode Wiki22.pngIntelligent Mail barcodeDiscrete4 bar heightsUnited States Postal Service, replaces both POSTNET and PLANET symbols (formerly named OneCode)\n' + '
ITF-14.svgITF-14ContinuousTwoNon-retail packaging levels, GS1-approved – is just an Interleaved 2/5 Code (ISO/IEC 16390) with a few additional specifications, according to the GS1 General Specifications\n' + '
ITF-6 barcode.svgITF-6ContinuousTwoInterleaved 2 of 5 barcode to encode an addon to ITF-14 and ITF-16 barcodes. The code is used to encode additional data such as items quantity or container weight\n' + '
EAN-13-5901234123457.svgJANContinuousManyUsed in Japan, similar to and compatible with EAN-13 (ISO/IEC 15420)\n' + '
Japan Post barcode.pngJapan Post barcodeDiscrete4 bar heightsJapan Post\n' + '
KarTrak ACI codes.svgKarTrak ACIDiscreteColoured barsUsed in North America on railroad rolling equipment\n' + '
MSI-barcode.pngMSIContinuousTwoUsed for warehouse shelves and inventory\n' + '
Pharmacode example.svgPharmacodeDiscreteTwoPharmaceutical packaging (no international standard available)\n' + '
Planet Barcode Format.pngPLANETContinuousTall/shortUnited States Postal Service (no international standard available)\n' + '
Plessey barcode.svgPlesseyContinuousTwoCatalogs, store shelves, inventory (no international standard available)\n' + '
Canada Post d52.01 domestic barcode.pngPostBarDiscrete4 bar heightsCanadian Post office\n' + '
POSTNET BAR.svg POSTNET 1.svg POSTNET 2.svg POSTNET 3.svg POSTNET BAR.pngPOSTNETDiscreteTall/shortUnited States Postal Service (no international standard available)\n' + '
Address with RM4SCC barcode.svgRM4SCC / KIXDiscrete4 bar heightsRoyal Mail / PostNL\n' + '
Royal Mail mailmark C barcode.pngRM Mailmark CDiscrete4 bar heightsRoyal Mail\n' + '
Royal Mail mailmark L barcode.pngRM Mailmark LDiscrete4 bar heightsRoyal Mail\n' + '
Telepen barcode.pngTelepenContinuousTwoLibraries (UK)\n' + '
UPC A.svgUniversal Product Code (UPC-A and UPC-E)ContinuousManyWorldwide retail, GS1-approved – International Standard ISO/IEC 15420\n' + '
\n' + '

Matrix (2D) barcodes

\n' + '

A matrix code, also termed a 2D barcode or simply a 2D code, is a two-dimensional way to represent information. It is similar to a linear (1-dimensional) barcode, but can represent more data per unit area.\n' + '

\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '
ExampleNameNotes\n' + '
Ar code.pngAR CodeA type of marker used for placing content inside augmented reality applications. Some AR Codes can contain QR codes inside, so that content AR content can be linked to.[39] See also ARTag.\n' + '
Azteccodeexample.svgAztec CodeDesigned by Andrew Longacre at Welch Allyn (now Honeywell Scanning and Mobility). Public domain. – International Standard: ISO/IEC 24778\n' + '
A bCode matrix barcode encoding the identifier 1683\n' + 'bCode\n' + 'A barcode designed for the study of insect behavior.[40] Encodes an 11 bit identifier and 16 bits of read error detection and error correction information. Predominately used for marking honey bees, but can also be applied to other animals.\n' + '
\n' + 'BEEtag\n' + 'A 25 bit (5x5) code matrix of black and white pixels that is unique to each tag surrounded by a white pixel border and a black pixel border. The 25-bit matrix consists of a 15-bit identity code, and a 10-bit error check.[41] It is designed to be a low-cost, image-based tracking system for the study of animal behavior and locomotion.\n' + '
\n' + 'BeeTagg\n' + 'A 2D barcode with honeycomb structures suitable for mobile tagging and was developed by the Swiss company connvision AG.\n' + '
\n' + 'Bokode\n' + 'A type of data tag which holds much more information than a barcode over the same area. They were developed by a team led by Ramesh Raskar at the MIT Media Lab. The bokode pattern is a tiled series of Data Matrix codes.\n' + '
Boxing 4kv6 0.png\n' + 'Boxing\n' + 'A high-capacity 2D barcode is used on piqlFilm by Piql AS[42]\n' + '
Code 1Public domain. Code 1 is currently used in the health care industry for medicine labels and the recycling industry to encode container content for sorting.[43]\n' + '
Code 16K wikipedia.png\n' + 'Code 16K\n' + 'The Code 16K (1988) is a multi-row bar code developed by Ted Williams at Laserlight Systems (USA) in 1992. In the US and France, the code is used in the electronics industry to identify chips and printed circuit boards. Medical applications in the USA are well known. Williams also developed Code 128, and the structure of 16K is based on Code 128. Not coincidentally, 128 squared happened to equal 16,000 or 16K for short. Code 16K resolved an inherent problem with Code 49. Code 49's structure requires a large amount of memory for encoding and decoding tables and algorithms. 16K is a stacked symbology.[44][45]\n' + '
ColorCodeColorZip[46] developed colour barcodes that can be read by camera phones from TV screens; mainly used in Korea.[47]\n' + '
Color Construct CodeColor Construct Code is one of the few barcode symbologies designed to take advantage of multiple colors.[48][49]\n' + '
PhotoTAN mit Orientierungsmarkierungen.svgCronto Visual Cryptogram\n' + 'The Cronto Visual Cryptogram (also called photoTAN) is a specialized color barcode, spun out from research at the University of Cambridge by Igor Drokov, Steven Murdoch, and Elena Punskaya.[50] It is used for transaction signing in e-banking; the barcode contains encrypted transaction data which is then used as a challenge to compute a transaction authentication number using a security token.[51]\n' + '
CyberCodeFrom Sony.\n' + '
d-touchreadable when printed on deformable gloves and stretched and distorted[52][53]\n' + '
DataGlyphsFrom Palo Alto Research Center (also termed Xerox PARC).[54]\n' + '

Patented.[55]\n' + 'DataGlyphs can be embedded into a half-tone image or background shading pattern in a way that is almost perceptually invisible, similar to steganography.[56][57]\n' + '

\n' + '
Datamatrix.svgData MatrixFrom Microscan Systems, formerly RVSI Acuity CiMatrix/Siemens. Public domain. Increasingly used throughout the United States. Single segment Data Matrix is also termed Semacode. – International Standard: ISO/IEC 16022.\n' + '
Datastrip CodeFrom Datastrip, Inc.\n' + '
\n' + 'Digimarc Barcode\n' + 'The Digimarc Barcode is a unique identifier, or code, based on imperceptible patterns that can be applied to marketing materials, including packaging, displays, ads in magazines, circulars, radio and television[58]\n' + '
digital paperpatterned paper used in conjunction with a digital pen to create handwritten digital documents. The printed dot pattern uniquely identifies the position coordinates on the paper.\n' + '
DotCode Wikipedia.pngDotCodeStandardized as AIM Dotcode Rev 3.0. Public domain. Used to track individual cigarette and pharmaceutical packages.\n' + '
Dot Code AAlso known as Philips Dot Code.[59] Patented in 1988.[60]\n' + '
\n' + 'DWCode\n' + 'Introduced by GS1 US and GS1 Germany, the DWCode is a unique, imperceptible data carrier that is repeated across the entire graphics design of a package[61]\n' + '
Example of an EZcode.EZcodeDesigned for decoding by cameraphones;[62] from ScanLife.[63]\n' + '
Han Xin 2D Barcode.svgHan Xin BarcodeBarcode designed to encode Chinese characters introduced by Association for Automatic Identification and Mobility in 2011.\n' + '
High Capacity Color Barcode Tag.svgHigh Capacity Color BarcodeHCCB was developed by Microsoft; licensed by ISAN-IA.\n' + '
HueCodeFrom Robot Design Associates. Uses greyscale or colour.[64]\n' + '
InterCodeFrom Iconlab, Inc. The standard 2D barcode in South Korea. All 3 South Korean mobile carriers put the scanner program of this code into their handsets to access mobile internet, as a default embedded program.\n' + '
\n' + '

JAB-code.png\n' + '

\n' + '
JAB Code\n' + 'Just Another Bar Code is a colored 2D barcode. Square or rectangle. License free\n' + '
MaxiCode.svgMaxiCodeUsed by United Parcel Service. Now public domain.\n' + '
\n' + 'mCode\n' + 'Designed by NextCode Corporation, specifically to work with mobile phones and mobile services.[65] It is implementing an independent error detection technique preventing false decoding, it uses a variable-size error correction polynomial, which depends on the exact size of the code.[66]\n' + '
MMCCDesigned to disseminate high capacity mobile phone content via existing colour print and electronic media, without the need for network connectivity\n' + '
NexCode.pngNexCodeNexCode is developed and patented by S5 Systems.\n' + '
Nintendo e-Reader#Dot codeDeveloped by Olympus Corporation to store songs, images, and mini-games for Game Boy Advance on Pokémon trading cards.\n' + '
Better Sample PDF417.pngPDF417Originated by Symbol Technologies. Public domain. – International standard: ISO/IEC 15438\n' + '
Qode example.QodeAmerican proprietary and patented 2D barcode from NeoMedia Technologies, Inc.[63]\n' + '
QR code for mobile English Wikipedia.svgQR codeInitially developed, patented and owned by Denso Wave for automotive components management; they have chosen not to exercise their patent rights. Can encode Latin and Japanese Kanji and Kana characters, music, images, URLs, emails. De facto standard for Japanese cell phones. Used with BlackBerry Messenger to pick up contacts rather than using a PIN code. The most frequently used type of code to scan with smartphones. Public Domain. – International Standard: ISO/IEC 18004\n' + '
ScreencodeDeveloped and patented[67][68] by Hewlett-Packard Labs. A time-varying 2D pattern using to encode data via brightness fluctuations in an image, for the purpose of high bandwidth data transfer from computer displays to smartphones via smartphone camera input. Inventors Timothy Kindberg and John Collomosse, publicly disclosed at ACM HotMobile 2008.[69]\n' + '
Shotcode.pngShotCodeCircular barcodes for camera phones. Originally from High Energy Magic Ltd in name Spotcode. Before that most likely termed TRIPCode.\n' + '
Snapcode, also called Boo-R codeused by Snapchat, Spectacles, etc. US9111164B1[70][71][72]\n' + '
\n' + 'Snowflake Code\n' + 'A proprietary code developed by Electronic Automation Ltd. in 1981. It is possible to encode more than 100 numeric digits in a space of only 5mm x 5mm. User selectable error correction allows up to 40% of the code to be destroyed and still remain readable. The code is used in the pharmaceutical industry and has an advantage that it can be applied to products and materials in a wide variety of ways, including printed labels, ink-jet printing, laser-etching, indenting or hole punching.[44][73][74]\n' + '
SPARQCode-sample.gifSPARQCodeQR code encoding standard from MSKYNET, Inc.\n' + '
\n' + 'Trillcode\n' + 'Designed for mobile phone scanning.[75] Developed by Lark Computer, a Romanian company.[66]\n' + '
VOICEYEDeveloped and patented by VOICEYE, Inc. in South Korea, it aims to allow blind and visually impaired people to access printed information. It also claims to be the 2D barcode that has the world's largest storage capacity.\n' + '
\n' + '

Example images

\n' + '\n' + '

In popular culture

\n' + '

In architecture, a building in Lingang New City by German architects Gerkan, Marg and Partners incorporates a barcode design,[77] as does a shopping mall called Shtrikh-kod (Russian for barcode) in Narodnaya ulitsa ("People's Street") in the Nevskiy district of St. Petersburg, Russia.[78]\n' + '

In media, in 2011, the National Film Board of Canada and ARTE France launched a web documentary entitled Barcode.tv, which allows users to view films about everyday objects by scanning the product's barcode with their iPhone camera.[79][80]\n' + '

In professional wrestling, the WWE stable D-Generation X incorporated a barcode into their entrance video, as well as on a T-shirt.[81][82]\n' + '

In the TV series Dark Angel, the protagonist and the other transgenics in the Manticore X-series have barcodes on the back of their necks.\n' + '

In video games, the protagonist of the Hitman video game series has a barcode tattoo on the back of his head. Also, QR codes can be scanned for an extra mission on Watch Dogs.\n' + '

In the films Back to the Future Part II and The Handmaid's Tale, cars in the future are depicted with barcode licence plates.\n' + '

In the Terminator films, Skynet burns barcodes onto the inside surface of the wrists of captive humans (in a similar location to the WW2 concentration camp tattoos) as a unique identifier.\n' + '

In music, Dave Davies of The Kinks released a solo album in 1980, AFL1-3603, which featured a giant barcode on the front cover in place of the musician's head. The album's name was also the barcode number.\n' + '

The April 1978 issue of Mad Magazine featured a giant barcode on the cover, with the blurb "[Mad] Hopes this issue jams up every computer in the country...for forcing us to deface our covers with this yecchy UPC symbol from now on!"\n' + '

The 2018 videogame Judgment features QR Codes that protagonist Takayuki Yagami can photograph with his phone camera. These are mostly to unlock parts for Yagami's Drone.[83]\n' + '

Interactive Textbooks were first published by Harcourt College Publishers to Expand Education Technology with Interactive Textbooks.[84]\n' + '

\n' + '

Designed barcodes

\n' + '

Some brands integrate custom designs into barcodes (while keeping them readable) on their consumer products.\n' + '

\n' + '\n' + '

Hoaxes about barcodes

\n' + '

There was minor skepticism from conspiracy theorists, who considered barcodes to be an intrusive surveillance technology, and from some Christians, pioneered by a 1982 book The New Money System 666 by Mary Stewart Relfe, who thought the codes hid the number 666, representing the "Number of the Beast".[85] Old Believers, a separation of the Russian Orthodox Church, believe barcodes are the stamp of the Antichrist.[86] Television host Phil Donahue described barcodes as a "corporate plot against consumers".[87]\n' + '

\n' + '

See also

\n' + '\n' + '

References

\n' + '
\n' + '
    \n' + '
  1. ^ a b US patent 2612994 \n' + '
  2. \n' + '
  3. ^ "How Barcodes Work". Stuff You Should Know. 4 June 2019. Retrieved 5 June 2019.\n' + '
  4. \n' + '
  5. ^ a b Cranstone, Ian. "A guide to ACI (Automatic Car Identification)/KarTrak". Canadian Freight Cars A resource page for the Canadian Freight Car Enthusiast. Retrieved 26 May 2013.\n' + '
  6. \n' + '
  7. ^ Keyes, John (22 August 2003). "KarTrak". John Keyes Boston photoblogger. Images from Boston, New England, and beyond. John Keyes. Archived from the original on 10 March 2014. Retrieved 26 May 2013.\n' + '
  8. \n' + '
  9. ^ a b Roberts, Sam (11 December 2019). "George Laurer, Who Developed the Bar Code, Is Dead at 94". New York Times. Retrieved 13 December 2019.\n' + '
  10. \n' + '
  11. ^ Fox, Margalit (15 June 2011). "Alan Haberman, Who Ushered in the Bar Code, Dies at 81". New York Times.\n' + '
  12. \n' + '
  13. ^ G. F. (2 November 2017). "Why QR codes are on the rise". The Economist. Retrieved 5 February 2018.\n' + '
  14. \n' + '
  15. ^ Fishman, Charles (1 August 2001). "The Killer App – Bar None". American Way. Archived from the original on 12 January 2010. Retrieved 19 April 2010.\n' + '
  16. \n' + '
  17. ^ a b c d e f Seideman, Tony (Spring 1993), "Barcodes Sweep the World", Wonders of Modern Technology, archived from the original on 16 October 2016\n' + '
  18. \n' + '
  19. ^ Dunn, Peter (20 October 2015). "David Collins, SM '59: Making his mark on the world with bar codes". technologyreview.com. MIT. Retrieved 2 December 2019.\n' + '
  20. \n' + '
  21. ^ Graham-White, Sean (August 1999). "Do You Know Where Your Boxcar Is?". Trains. 59 (8): 48–53.\n' + '
  22. \n' + '
  23. ^ Laurer, George. "Development of the U.P.C. Symbol". Archived from the original on 25 September 2008.\n' + '
  24. \n' + '
  25. ^ Nelson, Benjamin (1997). Punched Cards To Bar Codes: A 200-year journey. Peterborough, N.H.: Helmers. ISBN 9780911261127.\n' + '
  26. \n' + '
  27. ^ a b Varchaver, Nicholas (31 May 2004). "Scanning the Globe". Fortune. Archived from the original on 14 November 2006. Retrieved 27 November 2006.\n' + '
  28. \n' + '
  29. ^ a b Selmeier, Bill (2009). Spreading the Barcode. Lulu. pp. 26, 214, 236, 238, 244, 245, 236, 238, 244, 245. ISBN 978-0-578-02417-2.\n' + '
  30. \n' + '
  31. ^ Rawsthorn, Alice (23 February 2010). "Scan Artists". New York Times. Retrieved 31 July 2015.\n' + '
  32. \n' + '
  33. ^ "World hails barcode on important birthday". ATN. 1 July 2014.\n' + '
  34. \n' + '
  35. ^ "A Short History of Bar Code". BarCode 1. Adams Communications. Retrieved 28 November 2011.\n' + '
  36. \n' + '
  37. ^ "Barcode". iWatch Systems. 2 May 2011. Retrieved 28 November 2011.\n' + '
  38. \n' + '
  39. ^ Oberfield, Craig. "QNotes Barcode System". US Patented #5296688. Quick Notes Inc. Retrieved 15 December 2012.\n' + '
  40. \n' + '
  41. ^ National Geographic, May 2010, page 30\n' + '
  42. \n' + '
  43. ^ Hecht, David L. (March 2001). "Printed Embedded Data Graphical User Interfaces" (PDF). IEEE Computer. Xerox Palo Alto Research Center. 34 (3): 47–55. doi:10.1109/2.910893. Archived from the original (PDF) on 3 June 2013.\n' + '
  44. \n' + '
  45. ^ Howell, Jon; Kotay, Keith (March 2000). "Landmarks for absolute localization". Dartmouth Computer Science Technical Report TR2000-364.\n' + '
  46. \n' + '
  47. ^ "IATA.org". IATA.org. 21 November 2011. Retrieved 28 November 2011.\n' + '
  48. \n' + '
  49. ^ "Paperbyte Bar Codes for Waduzitdo". Byte magazine. September 1978. p. 172.\n' + '
  50. \n' + '
  51. ^ "Nokia N80 Support". Nokia Europe. Archived from the original on 14 July 2011.\n' + '
  52. \n' + '
  53. ^ "package overview for mbarcode". Maemo.org. Retrieved 28 July 2010.\n' + '
  54. \n' + '
  55. ^ Sargent, Mikah (24 September 2017). "How to use QR codes in iOS 11". iMore. Retrieved 1 October 2017.\n' + '
  56. \n' + '
  57. ^ "15+ Best Barcode Scanner iPhone Applications". iPhoneness. 3 March 2017. Retrieved 1 October 2017.\n' + '
  58. \n' + '
  59. ^ David, H (28 November 2018), "Barcodes – Validation vs Verification in GS1", Labeling News, retrieved 6 June 2020\n' + '
  60. \n' + '
  61. ^ "Layman's Guide to ANSI, CEN, and ISO Barcode Print Quality Documents" (PDF). Association for Automatic Identification and Data Capture Technologies (AIM). 2002. Retrieved 23 November 2017.\n' + '
  62. \n' + '
  63. ^ Zieger, Anne (October 2003). "Retailer chargebacks: is there an upside? Retailer compliance initiatives can lead to efficiency". Frontline Solutions. Archived from the original on 8 July 2012.\n' + '
  64. \n' + '
  65. ^ a b c d e f g h i j k l Corp, Express. "Barcode Glossary | Express". Express Corp. Retrieved 11 December 2019.\n' + '
  66. \n' + '
  67. ^ Bar Code Verification Best Practice work team (May 2010). "GS1 DataMatrix: An introduction and technical overview of the most advanced GS1 Application Identifiers compliant symbology" (PDF). Global Standards 1. 1 (17): 34–36. Archived (PDF) from the original on 20 July 2011. Retrieved 2 August 2011.\n' + '
  68. \n' + '
  69. ^ GS1 Bar Code Verification Best Practice work team (May 2009). "GS1 Bar Code Verification for Linear Symbols" (PDF). Global Standards 1. 4 (3): 23–32. Retrieved 2 August 2011.\n' + '
  70. \n' + '
  71. ^ Garner, J (2019), Results of Data Matrix Barcode Testing for Field Applications, Oak Ridge National Laboratory, retrieved 6 June 2020\n' + '
  72. \n' + '
  73. ^ "Technical committees – JTC 1/SC 31 – Automatic identification and data capture techniques". ISO. Retrieved 28 November 2011.\n' + '
  74. \n' + '
  75. ^ Harmon, Craig K.; Adams, Russ (1989). Reading Between The Lines:An Introduction to Bar Code Technology. Peterborough, NH: Helmers. p. 13. ISBN 0-911261-00-1.\n' + '
  76. \n' + '
  77. ^ "AR Code Generator"\n' + '
  78. \n' + '
  79. ^ Gernat, Tim; Rao, Vikyath D.; Middendorf, Martin; Dankowicz, Harry; Goldenfeld, Nigel; Robinson, Gene E. (13 February 2018). "Automated monitoring of behavior reveals bursty interaction patterns and rapid spreading dynamics in honeybee social networks". Proceedings of the National Academy of Sciences. 115 (7): 1433–1438. doi:10.1073/pnas.1713568115. ISSN 0027-8424. PMC 5816157. PMID 29378954.\n' + '
  80. \n' + '
  81. ^ Combes, Stacey A.; Mountcastle, Andrew M.; Gravish, Nick; Crall, James D. (2 September 2015). "BEEtag: A Low-Cost, Image-Based Tracking System for the Study of Animal Behavior and Locomotion". PLOS ONE. 10 (9): e0136487. Bibcode:2015PLoSO..1036487C. doi:10.1371/journal.pone.0136487. ISSN 1932-6203. PMC 4558030. PMID 26332211.\n' + '
  82. \n' + '
  83. ^ https://github.com/piql/boxing\n' + '
  84. \n' + '
  85. ^ Adams, Russ (15 June 2009). "2-Dimensional Bar Code Page". Archived from the original on 7 July 2011. Retrieved 6 June 2011.\n' + '
  86. \n' + '
  87. ^ a b "2-Dimensional Bar Code Page". www.adams1.com. Retrieved 12 January 2019.\n' + '
  88. \n' + '
  89. ^ "Code 16K Specs" (PDF). www.gomaro.ch. Retrieved 12 January 2019.\n' + '
  90. \n' + '
  91. ^ "Colorzip.com". Colorzip.com. Retrieved 28 November 2011.\n' + '
  92. \n' + '
  93. ^ "Barcodes for TV Commercials". Adverlab. 31 January 2006. Retrieved 10 June 2009.\n' + '
  94. \n' + '
  95. ^ "About". Colour Code Technologies. Archived from the original on 29 August 2012. Retrieved 4 November 2012.\n' + '
  96. \n' + '
  97. ^ "Frequently Asked Questions". ColorCCode. Archived from the original on 21 February 2013. Retrieved 4 November 2012.\n' + '
  98. \n' + '
  99. ^ "New system to combat online banking fraud". University of Cambridge. 18 April 2013. Retrieved 21 January 2020.\n' + '
  100. \n' + '
  101. ^ Cronto Visual Transaction Signing, OneSpan, retrieved 6 December 2019\n' + '
  102. \n' + '
  103. ^ d-touch topological fiducial recognition, MIT, archived from the original on 2 March 2008.\n' + '
  104. \n' + '
  105. ^ d-touch markers are applied to deformable gloves, MIT, archived from the original on 21 June 2008.\n' + '
  106. \n' + '
  107. ^ See Xerox.com for details.\n' + '
  108. \n' + '
  109. ^ "DataGlyphs: Embedding Digital Data". Microglyphs. 3 May 2006. Retrieved 10 March 2014.\n' + '
  110. \n' + '
  111. ^ ""DataGlyph" Embedded Digital Data". Tauzero. Retrieved 10 March 2014.\n' + '
  112. \n' + '
  113. ^ "DataGlyphs". Xerox. Retrieved 10 March 2014.\n' + '
  114. \n' + '
  115. ^ "Better Barcodes, Better Business" (PDF).\n' + '
  116. \n' + '
  117. ^ Dot Code A at barcode.ro\n' + '
  118. \n' + '
  119. ^ Dot Code A Patent\n' + '
  120. \n' + '
  121. ^ "GS1 Germany and Digimarc Announce Collaboration to Bring DWCode to the German Market".\n' + '
  122. \n' + '
  123. ^ "Scanbuy". Retrieved 28 November 2011.\n' + '
  124. \n' + '
  125. ^ a b Steeman, Jeroen. "Online QR Code Decoder". Archived from the original on 9 January 2014. Retrieved 9 January 2014.\n' + '
  126. \n' + '
  127. ^ "BarCode-1 2-Dimensional Bar Code Page". Adams. Archived from the original on 3 November 2008. Retrieved 10 June 2009.\n' + '
  128. \n' + '
  129. ^ "Global Research Solutions – 2D Barcodes". grs.weebly.com. Retrieved 12 January 2019.\n' + '
  130. \n' + '
  131. ^ a b Kato, Hiroko; Tan, Keng T.; Chai, Douglas (8 April 2010). Barcodes for Mobile Devices. Cambridge University Press. ISBN 9781139487511.\n' + '
  132. \n' + '
  133. ^ \n' + '"US Patent 9270846: Content encoded luminosity modulation"\n' + '
  134. \n' + '
  135. ^ \n' + '"US Patent 8180163: Encoder and decoder and methods of encoding and decoding sequence information with inserted monitor flags"\n' + '
  136. \n' + '
  137. ^ \n' + '"Screen Codes: Visual Hyperlinks for Displays"\n' + '
  138. \n' + '
  139. ^ \n' + '"Snapchat is changing the way you watch snaps and add friends"\n' + '
  140. \n' + '
  141. ^ \n' + '"Snapchat Lets You Add People Via QR Snaptags Thanks To Secret Scan.me Acquisition"\n' + '
  142. \n' + '
  143. ^ \n' + '"How Snapchat Made QR Codes Cool Again"\n' + '
  144. \n' + '
  145. ^ 5825015, Chan, John Paul & GB, "United States Patent: 5825015 – Machine readable binary codes", issued 20 October 1998 \n' + '
  146. \n' + '
  147. ^ "US Patent 5825015". pdfpiw.uspto.gov. 20 October 1998. Retrieved 12 January 2019.\n' + '
  148. \n' + '
  149. ^ "Trillcode Barcode". Barcoding, Inc. 17 February 2009. Retrieved 12 January 2019.\n' + '
  150. \n' + '
  151. ^ (株)デンソーウェーブ, denso-wave.com (in Japanese) Copyright\n' + '
  152. \n' + '
  153. ^ Barcode Halls – gmp Archived 18 October 2011 at the Wayback Machine\n' + '
  154. \n' + '
  155. ^ "image". Peterburg2.ru. Retrieved 28 November 2011.\n' + '
  156. \n' + '
  157. ^ Lavigne, Anne-Marie (5 October 2011). "Introducing Barcode.tv, a new interactive doc about the objects that surround us". NFB Blog. National Film Board of Canada. Retrieved 7 October 2011.\n' + '
  158. \n' + '
  159. ^ Anderson, Kelly (6 October 2011). "NFB, ARTE France launch 'Bar Code'". Reelscreen. Retrieved 7 October 2011.\n' + '
  160. \n' + '
  161. ^ [1] Archived 16 March 2015 at the Wayback Machine\n' + '
  162. \n' + '
  163. ^ "Dx theme song 2009–2010". YouTube. 19 December 2009. Retrieved 10 March 2014.\n' + '
  164. \n' + '
  165. ^ Diego Agruello (27 June 2019). "Judgment QR code locations to upgrade Drone Parts explained • Eurogamer.net". Retrieved 3 August 2019.\n' + '
  166. \n' + '
  167. ^ "CueCat History". CueCat History. Retrieved 12 November 2019.\n' + '
  168. \n' + '
  169. ^ "What about barcodes and 666: The Mark of the Beast?". Av1611.org. 1999. Retrieved 14 March 2014.\n' + '
  170. \n' + '
  171. ^ Serafino, Jay (26 July 2018). "The Russian Family That Cut Itself Off From Civilization for More Than 40 Years". Mental Floss. Retrieved 6 May 2020.\n' + '
  172. \n' + '
  173. ^ Bishop, Tricia (5 July 2004). "UPC bar code has been in use 30 years". SFgate.com. Archived from the original on 23 August 2004. Retrieved 22 December 2009.\n' + '
  174. \n' + '
\n' + '

Further reading

\n' + '
\n' + '
  • Automating Management Information Systems: Barcode Engineering and Implementation – Harry E. Burke, Thomson Learning, ISBN 0-442-20712-3
  • \n' + '
  • Automating Management Information Systems: Principles of Barcode Applications – Harry E. Burke, Thomson Learning, ISBN 0-442-20667-4
  • \n' + '
  • The Bar Code Book – Roger C. Palmer, Helmers Publishing, ISBN 0-911261-09-5, 386 pages
  • \n' + '
  • The Bar Code Manual – Eugene F. Brighan, Thompson Learning, ISBN 0-03-016173-8
  • \n' + '
  • Handbook of Bar Coding Systems – Harry E. Burke, Van Nostrand Reinhold Company, ISBN 978-0-442-21430-2, 219 pages
  • \n' + '
  • Information Technology for Retail:Automatic Identification & Data Capture Systems – Girdhar Joshi, Oxford University Press, ISBN 0-19-569796-0, 416 pages
  • \n' + '
  • Lines of Communication – Craig K. Harmon, Helmers Publishing, ISBN 0-911261-07-9, 425 pages
  • \n' + '
  • Punched Cards to Bar Codes – Benjamin Nelson, Helmers Publishing, ISBN 0-911261-12-5, 434 pages
  • \n' + '
  • Revolution at the Checkout Counter: The Explosion of the Bar Code – Stephen A. Brown, Harvard University Press, ISBN 0-674-76720-9
  • \n' + '
  • Reading Between The Lines – Craig K. Harmon and Russ Adams, Helmers Publishing, ISBN 0-911261-00-1, 297 pages
  • \n' + '
  • The Black and White Solution: Bar Code and the IBM PC – Russ Adams and Joyce Lane, Helmers Publishing, ISBN 0-911261-01-X, 169 pages
  • \n' + '
  • Sourcebook of Automatic Identification and Data Collection – Russ Adams, Van Nostrand Reinhold, ISBN 0-442-31850-2, 298 pages
  • \n' + '
  • Inside Out: The Wonders of Modern Technology – Carol J. Amato, Smithmark Pub, ISBN 0831746572, 1993
\n' + '
\n' + '

External links

\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '
\n' + '
================================================ FILE: aquila/txt_transform/test.py ================================================ import requests import json def process_html (html, url): payload = { "html": html, "url": url } headers = { 'Content-Type': 'application/json' } # conn.request("POST", "localhost:5009/process", json.dumps(payload), headers) response = requests.request("POST", "http://localhost:5009/process", headers=headers, data=json.dumps(payload)) # res = conn.getresponse() # data = response.text() if response.status_code == 200: return response.json() else: return None def trim_content (html): payload = { "html": html } headers = { 'Content-Type': 'application/json' } # conn.request("POST", "localhost:5009/process", json.dumps(payload), headers) response = requests.request("POST", "http://localhost:5008/process", headers=headers, data=json.dumps(payload)) # res = conn.getresponse() # data = response.text() if response.status_code == 200: return response.json() else: return None if __name__ == "__main__": readablity = process_html("

Bitcoin

Bitcoin is invented by satoshi nakomoto


it is powered by mining

", "https://google.com") print(readablity) content_lines = trim_content(readablity["data"]["content"]) if content_lines["success"] == True: print(content_lines["result"]) ================================================ FILE: aquila/view/.dockerignore ================================================ node_modules .env* .next ================================================ FILE: aquila/view/.eslintrc.json ================================================ { "extends": "next/core-web-vitals" } ================================================ FILE: aquila/view/@types/next-auth.d.ts ================================================ import { DefaultSession } from "next-auth"; declare module "next-auth" { interface User { customerId: string; firstName: string; lastName: string; createdAt: string; accountStatus: string; token: string; } interface Session { user: { customerId: string; firstName: string; lastName: string; createdAt: string; accountStatus: string; token: string; } & DefaultSession } } declare module "next-auth/jwt" { interface JWT { customerId: string; firstName: string; lastName: string; createdAt: string; accountStatus: string; token: string; } } ================================================ FILE: aquila/view/Dockerfile ================================================ FROM node:16-alpine as builder ARG NEXTAUTH_URL ARG NEXTAUTH_SECRET ARG NEXT_PUBLIC_AQUILA_API_URL ARG NEXT_PUBLIC_BASE_URL ARG NODE_ENV=production WORKDIR /app COPY . . ENV NEXTAUTH_URL=$NEXTAUTH_URL ENV NEXTAUTH_SECRET=$NEXTAUTH_SECRET ENV NEXT_PUBLIC_AQUILA_API_URL=$NEXT_PUBLIC_AQUILA_API_URL ENV NEXT_PUBLIC_BASE_URL=$NEXT_PUBLIC_BASE_URL RUN yarn --frozen-lockfile RUN NODE_ENV=${NODE_ENV} yarn build FROM node:16-alpine WORKDIR /app COPY --from=builder /app/next.config.js ./ COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static ENV NEXT_TELEMETRY_DISABLED 1 EXPOSE 3000 CMD ["node", "server.js"] ================================================ FILE: aquila/view/README.md ================================================ # Aquila-Network-UI # ⚠️ Server and other Credentials are in workflow file. Never Make this Repo Public ## Getting Started First, run the development server: ```bash npm run dev # or yarn dev ``` ## Docker Deploy ```bash docker build --build-arg NEXTAUTH_URL=http://localhost:3000 --build-arg NEXTAUTH_SECRET=secret --build-arg NEXT_PUBLIC_AQUILA_API_URL=https://aquila.api/url --build-arg NEXT_PUBLIC_BASE_URL=http://localhost:3000 -t aquila-network/aquila-network-ui . ``` ================================================ FILE: aquila/view/components/hoc/InitComponent.tsx ================================================ import { signOut, useSession, getSession } from "next-auth/react"; import { FC, useEffect } from "react"; import { useAppDispatch } from "../../store"; import { signIn, signOut as signOutAction } from "../../store/slices/auth"; interface InitCompoentProps { children: React.ReactNode; } const InitComponent: FC = (props) => { const { status, data} = useSession(); const dispatch = useAppDispatch(); useEffect(() => { if(status === 'authenticated') { dispatch(signIn({ token: data.user.token, accountStatus: data.user.accountStatus, customer: { customerId: data.user.customerId, firstName: data.user.firstName, lastName: data.user.lastName, createdAt: data.user.createdAt } })) } if(status === 'unauthenticated') { dispatch(signOutAction()) } }, [status, dispatch, data]) return ( <> {props.children} ) } export default InitComponent; ================================================ FILE: aquila/view/components/layout/base/baseLayout.tsx ================================================ import { FC } from 'react'; import { ProgressLoaderProvider } from '../../ui/progressLoader/ProgressLoader'; interface BaseLayoutProps { children: React.ReactNode; } const BaseLayout: FC = (props) => { return ( <> {props.children} ) } export default BaseLayout; ================================================ FILE: aquila/view/components/layout/boxCenter/BoxCenterLayout.module.scss ================================================ .box-center-layout { display: flex; align-items: center; justify-content: center; height: 100vh; &__box { background: #fff; width: 350px; padding: 20px; border-radius: 3px; box-shadow: rgba(149, 157, 165, 0.2) 0px 8px 24px; } &__box-header { display: flex; justify-content: center; } &__box-header-logo { cursor: pointer; width: 80px; } } ================================================ FILE: aquila/view/components/layout/boxCenter/BoxCenterLayout.tsx ================================================ import Img from 'next/image'; import Link from 'next/link'; import Logo from '../../../public/img/logo.png'; import BaseLayout from '../base/baseLayout'; import classes from './BoxCenterLayout.module.scss'; const BoxCenterLayout = (props: any) => { return(
{props.children}
) } export default BoxCenterLayout; ================================================ FILE: aquila/view/components/layout/childLayout/settings/Header.module.scss ================================================ .header { display: flex; gap: 15px; align-items: center; padding: 10px 0px; &__info { color: #202A38; } &__info-title { margin: 0px; font-weight: 500; } &__info-desc { margin: 0px; font-size: 14px; color: #415572; } } ================================================ FILE: aquila/view/components/layout/childLayout/settings/Header.tsx ================================================ import Avatar from 'boring-avatars'; import { FC } from 'react'; import { AppState } from '../../../../store'; import classes from './Header.module.scss'; interface HeaderProps { authState: AppState["auth"]; } const Header: FC = (props) => { const name = `${props.authState.customer?.firstName} ${props.authState.customer?.lastName}`; return (

{name}

Your account

); } export default Header; ================================================ FILE: aquila/view/components/layout/childLayout/settings/SettingsLayout.module.scss ================================================ .settings { width: 900px; margin: auto; margin-top: 20px; &__header { } &__body { display: flex; margin-top: 20px; } &__sidebar { width: 250px; } &__content { width: 100%; } } ================================================ FILE: aquila/view/components/layout/childLayout/settings/SettingsLayout.tsx ================================================ import { FC } from "react"; import { AppState } from "../../../../store"; import MainLayout from "../../main/MainLayout"; import Header from "./Header"; import classes from './SettingsLayout.module.scss'; import Sidebar from "./Sidebar"; interface SettingsLayoutProps { children: React.ReactNode; authState: AppState["auth"]; } const SettingsLayout: FC = (props) => { return (
{props.children}
); } export default SettingsLayout; ================================================ FILE: aquila/view/components/layout/childLayout/settings/Sidebar.module.scss ================================================ .sidebar { list-style: none; margin: 0px; padding: 0px; &__item { padding: 10px 5px; } &__item-link { font-size: 16px; text-decoration: none; color: #202A38; transition: all .3s; } &__item-link:hover { color: #0FBD86; } } ================================================ FILE: aquila/view/components/layout/childLayout/settings/Sidebar.tsx ================================================ import Link from "next/link"; import classes from "./Sidebar.module.scss"; const Sidebar = () => { return ( ) } export default Sidebar; ================================================ FILE: aquila/view/components/layout/main/AddLink.module.scss ================================================ .add-link { max-width: 500px; box-sizing: border-box; padding: 10px; box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px; border: 1px solid #fff; position: relative; top: 30px; left: 50%; margin-left: -250px; background: #fff; border-radius: 3px; &__close { position: absolute; top: 5px; right: 5px; font-size: 24px; cursor: pointer; color: #415572; } &__close:hover { color: #202A38; } &__header { padding: 10px; } &__header-title { text-align: center; font-size: 20px; font-weight: 500; color: #202A38; } &__body { padding: 0px 10px; margin-bottom: 10px; } &__form-item { margin-top: 15px; } &__form-control { width: 100%; box-sizing: border-box; padding: 5px; outline: none; border: 1px solid #202A38; border-radius: 3px; } &__form-control-error { color: #F87272; margin-top: 5px; font-size: 12px; } &__form-btn { outline: none; background: #fff; padding: 5px 10px; border: 1px solid #202A38; border-radius: 3px; color: #202A38; transition: all .3s; cursor: pointer; } &__form-btn:hover { color: #0FBD86; border-color: #0FBD86; } &__form-btn:disabled { color: #AEBCD1; border-color: #AEBCD1;; } } ================================================ FILE: aquila/view/components/layout/main/AddLink.tsx ================================================ import { FC, useEffect } from 'react'; import { IoCloseCircleOutline } from 'react-icons/io5'; import { useForm } from 'react-hook-form'; import { } from 'react' import Modal from '../../ui/modal/Modal'; import classes from './AddLink.module.scss'; import { AppState } from '../../../store'; interface AddLinkFromData { url: string; } interface AddLinkProps { onClose: Function; onSubmitAddLink: Function; addLinkState: AppState["addLink"]; } const AddLink: FC = ({ onClose, onSubmitAddLink, addLinkState}) => { const { register, getValues, setError, formState: { errors }, reset} = useForm(); useEffect(() => { if(addLinkState.errors) { if(addLinkState.errors.url) { setError("url", { type: 'custom', message: addLinkState.errors.url }); } } }, [addLinkState.errors, setError]) const onSubmitHandler = async (e: any) => { e.preventDefault(); var data = getValues(); const status = await onSubmitAddLink(data); if(status) { reset(); onClose(); } } return (
e.stopPropagation()}> onClose()} className={classes["add-link__close"]}>

Add Link

{errors.url &&

{errors.url.message}

}
) } export default AddLink; ================================================ FILE: aquila/view/components/layout/main/Footer.module.scss ================================================ .footer { padding: 20px 0px; text-align: center; box-sizing: border-box; &__icon-menu { list-style: none; display: flex; justify-content: center; gap: 20px; margin: 20px 0px; } &__icon-menu-link { font-size: 25px; color: #202A38; transition: all .3s; } &__icon-menu-link:hover { color: #0FBD86; } } .footer-banner { position: fixed; bottom: 0px; left: 0px; right: 0px; background: #F87272; color: #fff; padding: 10px 5px; text-align: center; &__close-btn { background: none; border: none; outline: none; color: #fff; position: absolute; top: 5px; right: 5px; cursor: pointer; font-size: 16px; } &__text { font-size: 14px; } } ================================================ FILE: aquila/view/components/layout/main/Footer.tsx ================================================ import { FC, useState } from 'react'; import Link from 'next/link'; import { FaGithub, FaYoutube, FaMedium } from 'react-icons/fa'; import { FiX } from 'react-icons/fi'; import classes from './Footer.module.scss'; import moment from 'moment'; interface FooterProps { signedInUser: { firstName: string; lastName: string; createdAt: string; } | null; accountStatus: string | null; } const Footer: FC = (props) => { const [showBanner, setShowBanner] = useState(true); const { signedInUser, accountStatus } = props; let accountExpiryData; if(signedInUser) { accountExpiryData = moment(signedInUser.createdAt).add(14, 'days').format('Mo MMMM'); } return ( <> {accountStatus === 'TEMPORARY' && showBanner &&

{`Your Account is temporary and will be deleted automatically on ${accountExpiryData}.`}Please Activate you account

}

Copyright © 2022 - All right reserved by Aquila Network

); } export default Footer; ================================================ FILE: aquila/view/components/layout/main/Header.module.scss ================================================ .header { padding: 20px 0px; &__container { margin: auto; display: flex; justify-content: space-between; align-items: center; } &__brand-contaner { order: 1; } &__brand-link { display: flex; text-decoration: none; align-items: center; } &__brand-img-container { width: 50px; } &__brand-text { font-size: 16px; font-weight: 500; color: #202A38; } &__nav-container { order: 3; } &__nav-list-container { display: flex; list-style: none; align-items: center; gap: 10px; } &__nav-mobile-menu-close { display: none; border: none; outline: none; background: none; cursor: pointer; font-size: 24px; } &__nav-list-item { position: relative; } &__nav-list-item--mobile-menu { display: none; } &__nav-list-link { text-decoration: none; color: #202A38; transition: all .3; cursor: pointer; font-weight: 500 } &__nav-list-link--active { color: #0FBD86; } &__nav-list-link:hover { color: #0FBD86; } &__nav-dropdown { position: absolute; list-style: none; padding: 0px; background: #fff; min-width: 150px; right: 0px; margin-top: 10px; box-shadow: rgba(0, 0, 0, 0.1) 0px 10px 50px; // box-shadow: rgba(50, 50, 93, 0.25) 0px 13px 27px -5px, rgba(0, 0, 0, 0.3) 0px 8px 16px -8px; } &__nav-dropdown::before { content: ''; width: 0px; height: 0px; border: 10px solid #fff; border-left-color: transparent; border-right-color: transparent; border-top-color: transparent; position: absolute; top: -20px; right: 10px; } &__nav-dropdown-link { text-decoration: none; color: #202A38; font-size: 14px; display: block; padding: 10px 20px; } &__nav-dropdown-link:hover { background: #EFF2F6; } &__mobile-menu-btn-container { order: 2; display: none; } &__mobile-menu-btn { border: none; outline: none; background: none; cursor: pointer; font-size: 25px; } } @media only screen and (max-width: 768px) { .header { &__mobile-menu-btn-container { display: block; } &__nav-container { position:fixed; width: 100vw; height: 100vh; left: 0px; right: 0px; top: 0px; bottom: 0px; z-index: 200; background: #fff; display: none; } &__nav-container--show { display: block } &__nav-list-container { height: 100%; flex-direction: column; justify-content: center; } &__nav-mobile-menu-close { display: block; position: absolute; top: 20px; right: 20px; } &__nav-list-item { padding: 20px 0px; } &__nav-list-item--profile-item { display: none; } &__nav-list-item--mobile-menu { display: block; } &__mobile-menu-btn { } } } ================================================ FILE: aquila/view/components/layout/main/Header.tsx ================================================ import Image from "next/image"; import Link from "next/link"; import { FC, useState } from "react"; import Avatar from 'boring-avatars'; import { FiMenu, FiX } from 'react-icons/fi'; import Logo from '../../../public/img/logo.png'; import classes from './Header.module.scss'; import Container from "../../ui/layout/Container"; import Modal from "../../ui/modal/Modal"; import AddLink from "./AddLink"; import { useRouter } from "next/router"; import { AppState } from "../../../store"; interface HeaderProps { onSignOut: Function onSubmitAddLink: Function isAuth: boolean; addLinkState: AppState["addLink"]; signedInUser: { firstName: string; lastName: string; } | null; } const Header: FC = (props: HeaderProps) => { const { signedInUser, onSignOut } = props; const [toggleDropDown, setToggleDropDown] = useState(false); const [showAddLinkModal, setAddLinkModal] = useState(false); const [showMobileMenu, setShowMobileMenu] = useState(false); const router = useRouter(); const signOutHandler = (e: any) => { e.preventDefault(); onSignOut(); } const dropDownToggleHandler = (e: any) => { e.preventDefault(); setToggleDropDown(!toggleDropDown); } const addLinkHandler = (e: any) => { e.preventDefault(); setAddLinkModal(true); } const onCloseModalHander = (e: any) => { setAddLinkModal(false); } return ( <>
{showAddLinkModal && } ) } export default Header; ================================================ FILE: aquila/view/components/layout/main/MainLayout.module.scss ================================================ .main-layout { height: 100%; display: flex; flex-direction: column; justify-content: space-between; &__header--with-border { box-shadow: inset 0px -1px 1px #EFF2F6; } } ================================================ FILE: aquila/view/components/layout/main/MainLayout.tsx ================================================ import { signOut } from 'next-auth/react'; import { useRouter } from 'next/router'; import { FC, useEffect, useState } from 'react'; import { useAppDispatch, useAppSelector } from '../../../store'; import { selectAuth } from '../../../store/slices/auth'; import { addLink, AddLinkData, selectAddLink } from '../../../store/slices/bookmark/addLink'; import { toast, ToastOptions } from 'react-toastify'; import Footer from './Footer'; import Header from './Header'; import classes from './MainLayout.module.scss'; import { getLoggedInCustCollections, selectGetLoggedInCustCollections } from '../../../store/slices/collection/getLoggedInCustCollections'; import { Omit } from '@reduxjs/toolkit/dist/tsHelpers'; import { useProgressLoader } from '../../ui/progressLoader/ProgressLoader'; interface MainLayoutProps { children: React.ReactNode; headerBorder?: boolean; } const MainLayout: FC = ({ children, headerBorder = true}) => { const router = useRouter(); const authState = useAppSelector(selectAuth); const addLinkState = useAppSelector(selectAddLink) const currentCustomerCollections = useAppSelector(selectGetLoggedInCustCollections); const dispatch = useAppDispatch(); const [signedInUser, setSignedInUser] = useState<{ firstName: string, lastName: string, createdAt: string} | null>(null); const { setLoader } = useProgressLoader(); useEffect(() => { if(authState.isSignedIn && authState.customer) { setSignedInUser({ firstName: authState.customer.firstName, lastName: authState.customer.lastName, createdAt: authState.customer.createdAt }) } }, [authState]); useEffect(() => { const toastOptions: ToastOptions = { position: "top-center", hideProgressBar: true, } if(authState.isSignedIn) { dispatch(getLoggedInCustCollections()) .unwrap() .catch((e) => { toast(e.message, { ...toastOptions, type: "error"}); }) } }, [authState, dispatch]); const signOutHandler = async () => { await signOut(); router.push("/sign-in"); } const onSubmitAddLinkHandler = async (data: Omit) => { setLoader(true); const toastOptions: ToastOptions = { position: "top-center", hideProgressBar: true, } if(currentCustomerCollections.collecitons && currentCustomerCollections.collecitons[0]) { const formData = { url: data.url, collectionId: currentCustomerCollections.collecitons[0].id } try { await dispatch(addLink(formData)).unwrap(); toast("Link added successfully", { ...toastOptions, type: "success"}) setLoader(false); return true; }catch(e) { if(e instanceof Error) { toast(e.message, { ...toastOptions, type: "error"}); } setLoader(false); return false; } } } return (
{children}
) } export default MainLayout; ================================================ FILE: aquila/view/components/pages/account/editProfile/EditProfileForm.module.scss ================================================ .edit-profile-form { &__form-group { margin-bottom: 15px; } &__form-label { display: block; margin-bottom: 5px; font-size: 18px; } &__form-control { width: 100%; padding: 8px 5px; box-sizing: border-box; } &__form-control-error { color: #F87272; margin-top: 5px; font-size: 12px; } &__form-btn { padding: 5px 10px; background: #fff; outline: none; border-radius: 3px; font-size: 16px; transition: all .3s; border: 1px solid #202A38; cursor: pointer; display: flex; align-items: center; } &__form-btn:hover { border-color: #36D399; color: #36D399; } &__form-btn:disabled { border-color: #6D87AC; color: #6D87AC; cursor:not-allowed; } } ================================================ FILE: aquila/view/components/pages/account/editProfile/EditProfileForm.tsx ================================================ import { FC, useEffect } from 'react'; import { useForm } from 'react-hook-form'; import { AppState } from '../../../../store'; import { Customer } from '../../../../store/slices/types/Customer'; import classes from './EditProfileForm.module.scss'; interface EditProfileFormProps { customer: Customer; accountStatus: string | null; onSubmit: Function; activateCustomerState: AppState["activateCustomer"]; updateCustomerState: AppState["updateCustomer"]; } interface EditProfileFormData { firstName: string; lastName: string; email: string; desc: string; lightningAddress: string; } const EditProfileForm: FC = ({customer, accountStatus, onSubmit, activateCustomerState, updateCustomerState}) => { const { register, getValues, setError, reset, formState: { errors } } = useForm({ defaultValues: { firstName: customer.firstName, lastName: customer.lastName, email: customer.email, lightningAddress: customer.lightningAddress, desc: customer.desc } }); useEffect(() => { if(updateCustomerState.errors) { if(updateCustomerState.errors.firstName) { setError("firstName", { type: 'custom', message: updateCustomerState.errors.firstName }); } if(updateCustomerState.errors.lastName) { setError("lastName", { type: 'custom', message: updateCustomerState.errors.lastName }); } if(updateCustomerState.errors.email) { setError("email", { type: 'custom', message: updateCustomerState.errors.email }); } if(updateCustomerState.errors.desc) { setError("desc", { type: 'custom', message: updateCustomerState.errors.desc }); } } }, [updateCustomerState.errors, setError]) useEffect(() => { if(activateCustomerState.errors) { if(activateCustomerState.errors.firstName) { setError("firstName", { type: 'custom', message: activateCustomerState.errors.firstName }); } if(activateCustomerState.errors.lastName) { setError("lastName", { type: 'custom', message: activateCustomerState.errors.lastName }); } if(activateCustomerState.errors.email) { setError("email", { type: 'custom', message: activateCustomerState.errors.email }); } if(activateCustomerState.errors.desc) { setError("desc", { type: 'custom', message: activateCustomerState.errors.desc }); } } }, [activateCustomerState.errors, setError]) const submitHandler = async (e: any) => { const data = getValues(); e.preventDefault(); const status = await onSubmit(data) if(status){ reset(undefined, { keepValues: true}); } } return (
{errors.firstName &&

{errors.firstName.message}

}
{errors.lastName &&

{errors.lastName.message}

}
{errors.email &&

{errors.email.message}

}

Get a Lightning address ?

{errors.lightningAddress &&

{errors.lightningAddress.message}

}
{errors.desc &&

{errors.desc.message}

}
) } export default EditProfileForm; ================================================ FILE: aquila/view/components/pages/account/editProfile/EditProfileHeader.module.scss ================================================ .header { &__title { font-size: 22px; color: #202A38; } } ================================================ FILE: aquila/view/components/pages/account/editProfile/EditProfileHeader.tsx ================================================ import classes from './EditProfileHeader.module.scss'; const EditProfileHeader = () => { return (

Edit Profile

) } export default EditProfileHeader; ================================================ FILE: aquila/view/components/pages/account/editProfile/EditProfileWrapper.module.scss ================================================ .edit-profile-wrapper { margin-top: 20px; &__alert-container { margin-bottom: 20px; } } ================================================ FILE: aquila/view/components/pages/account/editProfile/EditProfileWrapper.tsx ================================================ import { FC } from 'react'; import { Oval } from 'react-loader-spinner'; import { AppState } from '../../../../store'; import SettingsLayout from "../../../layout/childLayout/settings/SettingsLayout" import Alert from '../../../ui/alert/Alert'; import EditProfileForm from "./EditProfileForm" import EditProfileHeader from "./EditProfileHeader" import classes from './EditProfileWrapper.module.scss'; interface EditProfileWrapperProps { currentLoggedInCustomerState: AppState["getCurrentLoggedInCustomer"]; authState: AppState["auth"]; updateCustomerState: AppState["updateCustomer"]; activateCustomerState: AppState["activateCustomer"]; onSubmit: Function } const EditProfileWrapper: FC = (props) => { return (
{props.authState.accountStatus === "TEMPORARY" &&
} {props.currentLoggedInCustomerState.status === "pending" && } {props.currentLoggedInCustomerState.status === "failed" && } {props.currentLoggedInCustomerState.customer && }
) } export default EditProfileWrapper ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/CollectionBookmarksPageWrapper.module.scss ================================================ .collection-bookmarks { display: flex; gap: 50px; &__search-area { flex-basis: 900px; } &__search-bar { margin-top: 20px; width: 600px; } &__search-results { margin-top: 30px; } &__sidebar { padding-top: 20vh; flex: 1 1 0; } &__sidebar-close { display: none; } } @media only screen and (max-width: 1200px) { .collection-bookmarks { gap: 20px; } } @media only screen and (max-width: 992px) { .collection-bookmarks { flex-direction: column; &__sidebar { position: fixed; bottom: 0px; left: 0px; right: 0px; padding-top: 0px; } &__search-bar { width: 80%; } &__sidebar--close { display: none; } &__sidebar-close { display: block; position: absolute; background: none; font-size: 18px; outline: none; border: none; top: 25px; right: 5px; cursor: pointer; } } } ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/CollectionBookmarksPageWrapper.tsx ================================================ import { FC, useEffect, useState } from "react"; import { Oval } from 'react-loader-spinner'; import { FiX } from 'react-icons/fi'; import { AppState } from "../../../../../store"; import { Collection } from "../../../../../store/slices/types/Collection"; import { Customer } from "../../../../../store/slices/types/Customer"; import MainLayout from "../../../../layout/main/MainLayout" import Container from "../../../../ui/layout/Container"; import classes from "./CollectionBookmarksPageWrapper.module.scss"; import SearchBar from "./SearchBar"; import SearchPageProfile from "./SearchPageProfile"; import SearchResults from "./SearchResults"; import { AuthState } from "../../../../../store/slices/auth"; interface CollectionBookmarksPageWrapperProps { customer: Customer | null; collection: Collection | null; bookmarksState: AppState["getPublicBookmarksByCollectionId"]; onSearch: Function; onClickNextPage: Function; onClickPrevPage: Function; onSubscribe: Function; isCollectionSubscribed: boolean | null; isSignedIn: boolean | null; currentLoggedInCustomer: AuthState['customer'] | null; onUnsubscribe: Function; } const CollectionBookmarksPageWrapper: FC = (props) => { const { collection, customer, bookmarksState, onSearch, onClickNextPage, onClickPrevPage, onSubscribe, isCollectionSubscribed, isSignedIn, onUnsubscribe, currentLoggedInCustomer } = props; const [hasNext, setHasNext] = useState(false); const [hasPrev, setHasPrev] = useState(false); const [showSidebar, setShowSidebar] = useState(true); useEffect(() => { if(bookmarksState.currentPage && bookmarksState.totalPages) { if(bookmarksState.currentPage < bookmarksState.totalPages) { setHasNext(true); }else { setHasNext(false); } if(bookmarksState.currentPage > 1) { setHasPrev(true) }else { setHasPrev(false) } } }, [bookmarksState]) return (
{bookmarksState.status === "pending" && } {bookmarksState.bookmarks && }
{collection && customer && }
) } export default CollectionBookmarksPageWrapper; ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchBar.module.scss ================================================ .search-bar { padding: 10px 5px; &__container { display: flex; border-bottom: 1px solid #AEBCD1; padding: 10px 5px; } &__input { margin-right: 10px; width: 100%; border: none; outline: none; } &__input::placeholder { color: #AEBCD1; } &__btn { border: none; outline: none; background: transparent; } } ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchBar.tsx ================================================ import { FC, useRef } from 'react'; import { IoSearch } from 'react-icons/io5'; import classes from './SearchBar.module.scss'; interface SearchBarProps { onSearch: Function } const SearchBar: FC = (props) => { const { onSearch } = props; const searchValueRef = useRef(null); const onSubmitHandler = (e: any) => { e.preventDefault(); onSearch(searchValueRef.current?.value) } return (
) } export default SearchBar; ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchPageProfile.module.scss ================================================ .search-profile { padding: 15px; margin-top: 20px; border: 1px solid #CED7E3; background: #fff; border-radius: 3px; &__header { display: flex; justify-content: space-between; margin-bottom: 10px; } &__header-subscribe-btn { margin-top: 5px; background: transparent; border: 1px solid #8DA2BE; outline: none; border-radius: 3px; color: #2E3D51; transition: all .3s; cursor: pointer; } &__header-subscribe-btn:hover { color: #0FBD86; border-color: #0FBD86; } &__header-subscribe-btn:disabled { color: #CED7E3; border-color: #CED7E3; } &__header-avatar { width: 80px; } &__header-left { display: flex; flex-direction: column; gap: 10px; } &__header-name { color: #202A38; font-size: 25px; } &__body-desc { color: #415572; margin: 0px; font-size: 14px; margin-bottom: 20px; } &__body-bookmark-title { font-size: 14px; margin-bottom: 5px; color: #415572; } &__body-share-bookmark-box { overflow: hidden; background: #EFF2F6; position: relative; box-sizing:border-box; margin: 0px auto; padding: 3px 4px; border-radius: 3px; } &__body-share-bookmark-btn { position: absolute; right: 0px; top: 0px; bottom: 0px; font-size: 12px; padding: 5px; color: #fff; background: #3ABFF8; border-radius: 3px; cursor: pointer; transition: all .3s; } &__body-share-bookmark-btn:hover { background: #089DDE; } &__body-share-bookmark-link { width: 100%; margin: 0px; border: none; outline: none; background: transparent; font-size: 12px; color: #415572; } &__body-feedback-container { margin-top: 20px; } &__body-feedback-txt { color: #097654; text-decoration: none; font-size: 14px; transition: all .3s; } &__body-feedback-txt:hover { color: #0FBD86; } } @media only screen and (max-width: 992px) { .search-profile { padding-top: 25px; &__header-left { display: flex; flex-direction: row; align-items: center; gap: 10px; } &__header-avatar { width: 40px; } } } ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchPageProfile.tsx ================================================ import { VscFeedback } from 'react-icons/vsc'; import { toast } from 'react-toastify'; import Avatar from "boring-avatars"; import classes from './SearchPageProfile.module.scss'; import { Customer } from '../../../../../store/slices/types/Customer'; import { FC, useRef, useState } from 'react'; import { Collection } from '../../../../../store/slices/types/Collection'; interface SearchPageProfileProps { customer: Customer; collection: Collection; onSubscribe: Function; isSignedIn: boolean | null; isCurrentLoggedInCustProfile: boolean; isCollectionSubscribed: boolean | null; onUnsubscribe: Function } const SearchPageProfile: FC = ({ customer, collection, onSubscribe, isSignedIn, isCollectionSubscribed, onUnsubscribe, isCurrentLoggedInCustProfile}) => { const bookmarkShareLinkRef = useRef(null); const [isSubscribing, setIsSubscribing] = useState(false); const [isUnSubscribing, setIsUnSubscribing] = useState(false); const onClickCopyBtnHandler = () => { navigator.clipboard.writeText(bookmarkShareLinkRef.current?.value || ""); toast("Collection link copied", { position: "top-center"}); } const onClickSubscribeHandler = async (e: any) => { e.preventDefault() setIsSubscribing(true); await onSubscribe(collection.id) setIsSubscribing(false); } const onClickUnSubscribeHandler = async (e: any) => { e.preventDefault() setIsUnSubscribing(true); await onUnsubscribe(collection.id) setIsUnSubscribing(false); } return (

{`${customer.firstName} ${customer.lastName} ${isCurrentLoggedInCustProfile ? '(Me)' : ''}`}

{!(isSignedIn && isCollectionSubscribed) && } {isSignedIn && isCollectionSubscribed && }

{collection.desc}

Share this bookmark

Copy
) } export default SearchPageProfile; ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResultItem.module.scss ================================================ .search-result-item { border-bottom: 1px solid #CED7E3; padding-bottom: 15px; margin-bottom: 10px; word-wrap: break-word; &__title-link { font-size: 22px; text-decoration: none; margin-bottom: 2px; color: #202A38; transition: all .3s; cursor: pointer; } &__title-link:hover { color: #0FBD86; } &__meta-info { padding-bottom: 0px; margin-bottom: 0px; font-size: 12px; color: #8DA2BE; } &__site-link { font-size: 14px; color: #536E92; transition: all .3s; } &__site-link:hover { color: #9B5094; } &__site-desc { color: #415572; margin-top: 5px; margin-bottom: 3px; padding-bottom: 0px; } } ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResultItem.tsx ================================================ import moment from 'moment'; import { FC } from 'react'; import classes from './SearchResultItem.module.scss'; interface SearchResultItemProps { title: string; url: string; summary: string; createdAt: string; } const SearchResultItem: FC = (props) => { const { title, url, summary, createdAt } = props; return (

{title}

Updated {moment(createdAt).fromNow()}

{summary.length > 255 ? summary.substring(0, 255)+ '...' : summary}

{url}
); } export default SearchResultItem; ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResults.module.scss ================================================ .search-result { &__results { margin-top: 20px; } &__item { margin-bottom: 25px; } &__header-info-desc { color: #536E92; } &__pagination { margin-top: 15px; display: flex; gap: 10px; } &__pagination-link { font-size: 14px; text-decoration: none; background: #EFF2F6; color: #202A38; padding: 5px; transition: all .3s; border-radius: 3px; } &__pagination-link:hover { background: #CED7E3; } } ================================================ FILE: aquila/view/components/pages/collection/bookmark/CollectionBookmarks/SearchResults.tsx ================================================ import { FC } from "react"; import { Bookmark } from "../../../../../store/slices/types/Bookmark"; import SearchResultItem from "./SearchResultItem"; import classes from './SearchResults.module.scss'; interface SearchResultsProps { bookmarks: Bookmark[]; hasNext: boolean; hasPrev: boolean; onClickNextPage: Function; onClickPrevPage: Function; totalRecords: number | null; } const SearchResults: FC = (props) => { const { bookmarks, hasNext, hasPrev, onClickNextPage, onClickPrevPage, totalRecords } = props; const onNextPageHandler = (e: any) => { e.preventDefault(); onClickNextPage() } const onPrevPageHandler = (e:any) => { e.preventDefault(); onClickPrevPage() } return (
{totalRecords !== null ?

Received {totalRecords} results

: null}
{bookmarks.map((bookmark, index) => (
))}
{hasNext && Previous} {hasPrev && Next}
); } export default SearchResults; ================================================ FILE: aquila/view/components/pages/explore/Explore.module.scss ================================================ .explore { // width: 250px; padding: 20px 15px; border-radius: 3px; border: 1px solid #ddd; &__header { display: flex; align-items: center; margin-bottom: 20px; padding-bottom: 20px; border-bottom: 2px solid #ddd; } &__header-title { color: #202A38; font-size: 22px; font-weight: 500; text-decoration: none; margin: 0px; } &__header-title:hover { color: #0FBD86; } &__header-avatar { margin-right: 10px; } &__content-desc { color: #415572; font-size: 14px; } } // @media only screen and (max-width: 1200px) { // .explore { // width: 200px; // } // } // @media only screen and (max-width: 992px) { // .explore { // width: 225px; // } // } // @media only screen and (max-width: 992px) { // .explore { // width: 250px; // } // } ================================================ FILE: aquila/view/components/pages/explore/Explore.tsx ================================================ import Avatar from "boring-avatars"; import Link from "next/link"; import { FC } from "react"; import { Collection } from "../../../store/slices/types/Collection"; import classes from "./Explore.module.scss"; interface ExploreProps { collection: Collection } const Explore: FC = (props) => { return ( ) } export default Explore; ================================================ FILE: aquila/view/components/pages/explore/ExploreCategoryItem.module.scss ================================================ .explore-category-item { margin-top: 15px; &__title { margin-bottom: 20px; font-size: 24px; } &__explore-list { display: grid; grid-template-columns: repeat(4, 1fr); justify-items: center; gap: 30px; } } @media only screen and (max-width: 992px) { .explore-category-item { &__explore-list { grid-template-columns: repeat(3, 1fr); } } } @media only screen and (max-width: 768px) { .explore-category-item { &__explore-list { grid-template-columns: repeat(2, 1fr); } } } @media only screen and (max-width: 576px) { .explore-category-item { &__explore-list { grid-template-columns: 1fr; } } } ================================================ FILE: aquila/view/components/pages/explore/ExploreCategoryItem.tsx ================================================ import { FC } from "react"; import { Collection } from "../../../store/slices/types/Collection"; import Explore from "./Explore"; import classes from './ExploreCategoryItem.module.scss'; interface ExploreItemProps { collections: Collection[]; title: string; } const ExploreItem: FC = (props) => { return (

{props.title}

{props.collections.map((item, index) => { return (
) })}
) } export default ExploreItem; ================================================ FILE: aquila/view/components/pages/explore/ExploreCategoryList.module.scss ================================================ .explore-category-list { &__item { margin-bottom: 50px; } } ================================================ FILE: aquila/view/components/pages/explore/ExploreCategoryList.tsx ================================================ import { FC } from "react"; import { AppState } from "../../../store"; import ExploreCategoryItem from "./ExploreCategoryItem"; import classes from './ExploreCategoryList.module.scss'; interface ExploreCategoryListProps { featuredCollectionsState: AppState["getFeaturedCollections"]; publicCollectionsState: AppState["getAllPublicCollections"]; } const ExploreCategoryList: FC = (props) => { return (
{props.featuredCollectionsState.collections &&
} {props.publicCollectionsState.collections &&
} {/*
*/}
); } export default ExploreCategoryList; ================================================ FILE: aquila/view/components/pages/explore/ExplorePageWrapper.tsx ================================================ import { FC } from "react"; import { AppState } from "../../../store"; import MainLayout from "../../layout/main/MainLayout"; import Container from "../../ui/layout/Container"; import ExploreCategoryList from "./ExploreCategoryList"; interface ExplorePageWrapperProps { featuredCollectionsState: AppState["getFeaturedCollections"]; publicCollectionsState: AppState["getAllPublicCollections"]; } const ExplorePageWrapper: FC = (props) => { return ( ) } export default ExplorePageWrapper; ================================================ FILE: aquila/view/components/pages/home/HomePageWrapper.module.scss ================================================ .home-page { display: flex; gap: 50px; &__search-area { flex-basis: 800px; } &__search-bar { margin-top: 20px; width: 600px; } &__search-results { margin-top: 30px; } &__sidebar { padding-top: 20vh; flex: 1 1 0; } &__sidebar-close { display: none; } } @media only screen and (max-width: 1200px) { .home-page { gap: 20px; } } @media only screen and (max-width: 992px) { .home-page { flex-direction: column; &__sidebar { position: fixed; bottom: 0px; left: 0px; right: 0px; padding-top: 0px; } &__search-bar { width: 80%; } &__sidebar--close { display: none; } &__sidebar-close { display: block; position: absolute; background: none; font-size: 18px; outline: none; border: none; top: 25px; right: 5px; cursor: pointer; } } } ================================================ FILE: aquila/view/components/pages/home/HomePageWrapper.tsx ================================================ import { FC, useEffect, useState } from "react"; import { Oval } from 'react-loader-spinner'; import { FiX } from 'react-icons/fi'; import { AppState } from "../../../store"; import { Collection } from "../../../store/slices/types/Collection"; import { Customer } from "../../../store/slices/types/Customer"; import MainLayout from "../../layout/main/MainLayout" import Container from "../../ui/layout/Container"; import classes from "./HomePageWrapper.module.scss"; import SearchBar from "./SearchBar"; import SearchPageProfile from "./SearchPageProfile"; import SearchResults from "./SearchResults"; interface HomePageWrapperProps { customer: Customer | null; collection: Collection | null; bookmarksState: AppState["getLoggedInCustBookmarksByCollectionId"]; onSearch: Function; onClickNextPage: Function; onClickPrevPage: Function; isCollectionSubscribed: boolean | null; onSubscribe: Function; onUnsubscribe: Function; } const HomePageWrapper: FC = (props) => { const { collection, customer, bookmarksState, onSearch, onClickNextPage, onClickPrevPage, onSubscribe, onUnsubscribe, isCollectionSubscribed } = props; const [hasNext, setHasNext] = useState(false); const [hasPrev, setHasPrev] = useState(false); const [showSidebar, setShowSidebar] = useState(true); useEffect(() => { if(bookmarksState.currentPage && bookmarksState.totalPages) { if(bookmarksState.currentPage < bookmarksState.totalPages) { setHasNext(true); }else { setHasNext(false); } if(bookmarksState.currentPage > 1) { setHasPrev(true) }else { setHasPrev(false) } } }, [bookmarksState]) return (
{bookmarksState.status === "pending" && } {bookmarksState.bookmarks && }
{collection && customer && }
) } export default HomePageWrapper; ================================================ FILE: aquila/view/components/pages/home/SearchBar.module.scss ================================================ .search-bar { padding: 10px 5px; &__container { display: flex; border-bottom: 1px solid #AEBCD1; padding: 10px 5px; } &__input { margin-right: 10px; width: 100%; border: none; outline: none; } &__input::placeholder { color: #AEBCD1; } &__btn { border: none; outline: none; background: transparent; } } ================================================ FILE: aquila/view/components/pages/home/SearchBar.tsx ================================================ import { FC, useRef } from 'react'; import { IoSearch } from 'react-icons/io5'; import classes from './SearchBar.module.scss'; interface SearchBarProps { onSearch: Function } const SearchBar: FC = (props) => { const { onSearch } = props; const searchValueRef = useRef(null); const onSubmitHandler = (e: any) => { e.preventDefault(); onSearch(searchValueRef.current?.value) } return (
) } export default SearchBar; ================================================ FILE: aquila/view/components/pages/home/SearchPageProfile.module.scss ================================================ .search-profile { padding: 15px; margin-top: 20px; border: 1px solid #CED7E3; border-radius: 3px; background: #fff; &__header { display: flex; justify-content: space-between; margin-bottom: 10px; } &__header-subscribe-btn { margin-top: 5px; background: transparent; border: 1px solid #8DA2BE; outline: none; border-radius: 3px; color: #2E3D51; transition: all .3s; cursor: pointer; } &__header-subscribe-btn:hover { color: #0FBD86; border-color: #0FBD86; } &__header-avatar { width: 80px; } &__header-left { display: flex; flex-direction: column; gap: 10px; } &__header-name { color: #202A38; font-size: 25px; } &__body-desc { color: #415572; margin: 0px; font-size: 14px; margin-bottom: 20px; } &__body-bookmark-title { font-size: 14px; margin-bottom: 5px; color: #415572; } &__body-share-bookmark-box { overflow: hidden; background: #EFF2F6; position: relative; box-sizing:border-box; margin: 0px auto; padding: 3px 4px; border-radius: 3px; } &__body-share-bookmark-btn { position: absolute; right: 0px; top: 0px; bottom: 0px; font-size: 12px; padding: 5px; color: #fff; background: #3ABFF8; border-radius: 3px; cursor: pointer; transition: all .3s; } &__body-share-bookmark-btn:hover { background: #089DDE; } &__body-share-bookmark-link { width: 100%; margin: 0px; border: none; outline: none; background: transparent; font-size: 12px; color: #415572; } &__body-feedback-container { margin-top: 20px; } &__body-feedback-txt { color: #097654; text-decoration: none; font-size: 14px; transition: all .3s; } &__body-feedback-txt:hover { color: #0FBD86; } } @media only screen and (max-width: 992px) { .search-profile { padding-top: 25px; &__header-left { display: flex; flex-direction: row; align-items: center; gap: 10px; } &__header-avatar { width: 40px; } } } ================================================ FILE: aquila/view/components/pages/home/SearchPageProfile.tsx ================================================ import { VscFeedback } from 'react-icons/vsc'; import { toast } from 'react-toastify'; import Avatar from "boring-avatars"; import classes from './SearchPageProfile.module.scss'; import { Customer } from '../../../store/slices/types/Customer'; import { FC, useRef, useState } from 'react'; import { Collection } from '../../../store/slices/types/Collection'; interface SearchPageProfileProps { customer: Customer; collection: Collection; onSubscribe: Function; onUnsubscribe: Function; isCollectionSubscribed: boolean | null; } const SearchPageProfile: FC = ({ customer, collection, onSubscribe, onUnsubscribe, isCollectionSubscribed }) => { const bookmarkShareLinkRef = useRef(null); const [isSubscribing, setIsSubscribing] = useState(false); const [isUnSubscribing, setIsUnSubscribing] = useState(false); const OnClickCopyBtnHandler = () => { navigator.clipboard.writeText(bookmarkShareLinkRef.current?.value || ""); toast("Collection link copied", { position: "top-center"}); } const onClickSubscribeHandler = async (e: any) => { e.preventDefault() setIsSubscribing(true); await onSubscribe(collection.id) setIsSubscribing(false); } const onClickUnSubscribeHandler = async (e: any) => { e.preventDefault() setIsUnSubscribing(true); await onUnsubscribe(collection.id) setIsUnSubscribing(false); } return (

{`${customer.firstName} ${customer.lastName} (Me)`}

{!isCollectionSubscribed && } {isCollectionSubscribed && }

{collection.desc}

Share this bookmark

Copy
) } export default SearchPageProfile; ================================================ FILE: aquila/view/components/pages/home/SearchResultItem.module.scss ================================================ .search-result-item { border-bottom: 1px solid #CED7E3; padding-bottom: 15px; margin-bottom: 10px; word-wrap: break-word; &__title { margin-bottom: 2px; } &__title-link { font-size: 22px; color: #202A38; transition: all .3s; cursor: pointer; text-decoration: none; } &__title-link:hover { color: #0FBD86; } &__meta-info { padding-bottom: 0px; margin-bottom: 0px; font-size: 12px; color: #8DA2BE; } &__site-link { font-size: 14px; color: #536E92; transition: all .3s; } &__site-link:hover { color: #9B5094; } &__site-desc { color: #415572; margin-top: 5px; margin-bottom: 3px; padding-bottom: 0px; } } ================================================ FILE: aquila/view/components/pages/home/SearchResultItem.tsx ================================================ import { FC } from 'react'; import moment from 'moment'; import classes from './SearchResultItem.module.scss'; interface SearchResultItemProps { title: string; url: string; summary: string; createdAt: string; } const SearchResultItem: FC = (props) => { const { title, url, summary, createdAt } = props; return (

{title}

Updated {moment(createdAt).fromNow()}

{summary.length > 255 ? summary.substring(0, 255)+'...' : summary}

{url}
); } export default SearchResultItem; ================================================ FILE: aquila/view/components/pages/home/SearchResults.module.scss ================================================ .search-result { &__results { margin-top: 20px; } &__item { margin-bottom: 25px; } &__header-info-desc { color: #536E92; } &__pagination { margin-top: 15px; display: flex; gap: 10px; } &__pagination-link { font-size: 14px; text-decoration: none; background: #EFF2F6; color: #202A38; padding: 5px; transition: all .3s; border-radius: 3px; } &__pagination-link:hover { background: #CED7E3; } } ================================================ FILE: aquila/view/components/pages/home/SearchResults.tsx ================================================ import { FC } from "react"; import { Bookmark } from "../../../store/slices/types/Bookmark"; import SearchResultItem from "./SearchResultItem"; import classes from './SearchResults.module.scss'; interface SearchResultsProps { bookmarks: Bookmark[]; hasNext: boolean; hasPrev: boolean; onClickNextPage: Function; onClickPrevPage: Function; totalRecords: number | null; } const SearchResults: FC = (props) => { const { bookmarks, hasNext, hasPrev, onClickNextPage, onClickPrevPage, totalRecords } = props; const onNextPageHandler = (e: any) => { e.preventDefault(); onClickNextPage() } const onPrevPageHandler = (e:any) => { e.preventDefault(); onClickPrevPage() } return (
{totalRecords !== null ?

Received {totalRecords} results

: null}
{bookmarks.map((bookmark, index) => (
))}
{hasNext && Previous} {hasPrev && Next}
); } export default SearchResults; ================================================ FILE: aquila/view/components/pages/index/AllControl.module.scss ================================================ .all-control { display: flex; margin: auto; max-width: 1280; &__img-area { flex-basis: 50%; padding: 50px; box-sizing: border-box; } &__text-area { flex-basis: 50%; display: flex; flex-direction: column; justify-content: center; gap: 20px; padding: 50px; box-sizing: border-box; } &__title { color: #202A38; font-size: 25px; } &__desc { color: #415572; font-size: 20px; } &__desc-link { color: #0FBD86; text-decoration: none; } } @media only screen and (max-width: 768px) { .all-control { flex-direction: column-reverse; padding-left: 10px; padding-right: 10px; text-align: center; &__title { font-size: 18px; } &__desc { font-size: 16px; } &__img-area { width: 80%; margin-left: auto; margin-right: auto; } } } ================================================ FILE: aquila/view/components/pages/index/AllControl.tsx ================================================ import Image from "next/image"; import classes from "./AllControl.module.scss"; import AllControlImg from "../../../public/img/aquila-control.png"; const AllControl = () => { return (
All Control

You’ve got all the control

nstall a browser extension and you're ready to go. We've made our software Open Source for public scrutiny & trust

) } export default AllControl; ================================================ FILE: aquila/view/components/pages/index/Discover.module.scss ================================================ .discover { margin: auto; margin-top: 80px; display: flex; max-width: 1280px; &__text-area { flex-basis: 50%; padding: 50px; box-sizing: border-box; display: flex; flex-direction: column; justify-content: center; gap: 20px; } &__title { font-size: 25px; color: #202A38; } &__desc { font-size: 20px; color: #415572; } &__img-area { flex-basis: 50%; } } @media only screen and (max-width: 768px) { .discover { flex-direction: column; padding-left: 10px; padding-right: 10px; text-align: center; &__title { font-size: 18px; } &__desc { font-size: 16px; } &__img-area { width: 80%; margin-left: auto; margin-right: auto; } } } ================================================ FILE: aquila/view/components/pages/index/Discover.tsx ================================================ import Image from 'next/image'; import classes from './Discover.module.scss'; import DiscoverImg from '../../../public/img/aquila-discover.png'; const Discover = () => { return (

Discover new topics and people

Become a fan of your favorite curators. Maintain a pool of specialists and create a search engine out of it.

Discover new Topics in Aquila
) } export default Discover; ================================================ FILE: aquila/view/components/pages/index/Hero.module.scss ================================================ .hero { text-align: center; &__text-area { margin-top: 50px; } &__text-main-title { font-size: 70px; color: #202A38; } &__text-main-desc { font-size: 20px; margin: auto; margin-top: 30px; max-width: 600px; color: #536E92; } &__btn-area { margin-top: 50px; display: flex; justify-content: center; gap: 10px; } &__img-area { margin: auto; margin-top: 60px; max-width: 900px; } &__btn { border: none; background: #CED7E3; padding: 10px; border-radius: 3px; color: #fff; } &__btn--red { background: #F87272; } &__btn--blue { background: #3ABFF8; } &__footer-text { margin: auto; margin-top: 50px; max-width: 600px; border-top: 1px solid #ddd; border-bottom: 1px solid #ddd; padding: 40px 0px; font-size: 20px; color: #2E3D51; } } @media only screen and (max-width: 768px) { .hero { &__text-main-title { font-size: 45px; } &__text-main-desc { font-size: 18px; line-height: 30px; } } } @media only screen and (max-width: 576px) { .hero { padding-left: 10px; padding-right: 10px; &__text-main-title { font-size: 35px; } &__text-main-desc { font-size: 16px; line-height: 24px; } &__footer-text { font-size: 18px; } } } ================================================ FILE: aquila/view/components/pages/index/Hero.tsx ================================================ import Image from 'next/image'; import classes from './Hero.module.scss'; import HeroImg from '../../../public/img/aquila-hero.png'; const Hero = () => { return (

Human curated search engine

Build a curated list of websites, aggregate into searchable pool of ideas. Works with paywall-protected websites.

Hero Image

Aquila offers you a familiar search experience. Easily organize and share your curated list with anybody.

) } export default Hero; ================================================ FILE: aquila/view/components/pages/index/IndexPageWrapper.tsx ================================================ import MainLayout from "../../layout/main/MainLayout"; import AllControl from "./AllControl"; import Discover from "./Discover"; import Hero from "./Hero"; import Story from "./Story"; const IndexPageWrapper = () => { return ( ) } export default IndexPageWrapper; ================================================ FILE: aquila/view/components/pages/index/Story.module.scss ================================================ .story { background: #EFF2F6; padding: 100px 0px; text-align: center; &__title { font-size: 24px; color: #202A38; } &__link { text-decoration: none; color: #0FBD86; } } @media only screen and (max-width: 768px) { .story { padding: 50px 10px; &__title { font-size: 18px; } } } ================================================ FILE: aquila/view/components/pages/index/Story.tsx ================================================ import classes from './Story.module.scss'; const Story = () => { return (

Interested to read the story of Aquila Network?

) } export default Story; ================================================ FILE: aquila/view/components/pages/signIn/SignInForm.module.scss ================================================ .login-box { &__header { margin: 25px 0px; } &__title { font-size: 20px; color: #415572; text-align: center; } &__form-item { margin-bottom: 12px; } &__form-item--btn-container { margin-top: 20px; } &__form-label { display: block; color: hsl(214, 28%, 45%); margin-bottom: 4px; } &__form-control { color: #415572; width: 100%; padding: 8px 5px; border: 1px solid #536E92; border-radius: 3px; box-sizing: border-box; } &__form-btn { width: 100%; background: none; outline: none; border: 1px solid #415572; color: #415572; padding: 6px 0px; border-radius: 3px; transition: all .3s; } &__form-btn:hover { color: #0FBD86; border: 1px solid #0FBD86; } &__form-btn:disabled { color: #CED7E3; border-color: #CED7E3; } &__footer-text { font-size: 12px; } &__footer-link { color: #0FBD86; text-decoration: none; } &__footer-link:hover { color: #0DA575; } } ================================================ FILE: aquila/view/components/pages/signIn/SignInForm.tsx ================================================ import Link from 'next/link'; import { useRef, useState } from 'react'; import classes from './SignInForm.module.scss'; const SignInForm = (props:any) => { const secretKeyRef = useRef(null); const [isLoading, setIsLoading] = useState(false); const submitHandler = (e: any) => { e.preventDefault(); setIsLoading(true); props.onSignIn(secretKeyRef.current?.value) .then(() => { setIsLoading(false); }) } return (

Login

Don't have an account? Generate an account

) } export default SignInForm; ================================================ FILE: aquila/view/components/pages/signIn/SignInPageWrapper.tsx ================================================ import { FC } from "react"; import BoxCenterLayout from "../../layout/boxCenter/BoxCenterLayout"; import SignInForm from "./SignInForm"; interface SignInPageWrapperProps { onSignIn: Function } const SignInPageWrapper: FC = (props) => { return ( ) } export default SignInPageWrapper; ================================================ FILE: aquila/view/components/pages/signUp/SecretKey.module.scss ================================================ .secret-key { padding: 20px 0px; text-align: center; &__header { padding: 5px 0px; } &__header-title { font-size: 20px; color: #2E3D51; } &__body { margin-top: 20px; } &__body-title { font-size: 16px; color: #536E92; } &__key { margin-top: 10px; background-color: #EFF2F6; font-size: 12px; border-radius: 3px; display: flex; align-items: center; justify-content: space-between; } &__key-text { padding: 10px 5px; color: #536E92; } &__copy-key { padding: 0px 10px; align-self: stretch; align-content: center; font-size: 10px; border-radius: 5px; background: #3ABFF8; font-size: 18px; display: flex; align-items: center; color: #fff; cursor: pointer; transition: all .3s; } &__copy-key:hover { background: #089DDE; } &__info-text { background-color:#E7FDF6; font-size: 14px; margin-top: 25px; padding: 10px 5px; border-radius: 3px; color: #064732; } &__footer { margin-top: 25px; } &__footer-btn { text-decoration: none; color: #202A38; display: block; border: 1px solid #202A38; border-radius: 3px; padding: 6px 5px; } &__footer-btn:hover { color: #0FBD86; border: 1px solid #0FBD86; } } ================================================ FILE: aquila/view/components/pages/signUp/SecretKey.tsx ================================================ import Link from "next/link"; import { FC } from "react"; import { CgCopy } from 'react-icons/cg'; import { toast } from 'react-toastify'; import { AppState } from "../../../store"; import classes from './SecretKey.module.scss'; interface SecretKeyProps { signUpState: AppState["signUp"]; } const SecretKey: FC = ({ signUpState }) => { const copyToClipboardHandler = () => { navigator.clipboard.writeText(signUpState.customer?.secretKey || ""); toast("Content copied", { position: "top-center"}); } return (

Your Account has been generated

Your Secret key

{signUpState.customer?.secretKey}

Make sure to copy your secret key. Secret key is required to login next time

) } export default SecretKey; ================================================ FILE: aquila/view/components/pages/signUp/SignUpForm.module.scss ================================================ .signup-box { &__header { margin: 25px 0px; } &__title { font-size: 20px; color: #415572; text-align: center; } &__form-item { margin-bottom: 12px; } &__form-item--btn-container { margin-top: 30px; } &__form-label { display: block; color: hsl(214, 28%, 45%); margin-bottom: 4px; } &__form-control { color: #415572; width: 100%; padding: 8px 5px; border: 1px solid #536E92; border-radius: 3px; box-sizing: border-box; } &__form-error { margin-top: 5px; color: #F87272; font-size: 12px; } &__form-btn { width: 100%; background: none; outline: none; border: 1px solid #415572; color: #415572; padding: 6px 0px; border-radius: 3px; transition: all .3s; cursor: pointer; } &__form-btn:hover { color: #0FBD86; border: 1px solid #0FBD86; } &__form-btn:disabled { color: #CED7E3; border: 1px solid #CED7E3; cursor: not-allowed } &__footer-text { font-size: 12px; } &__footer-link { color: #0FBD86; text-decoration: none; } &__footer-link:hover { color: #0DA575; } } ================================================ FILE: aquila/view/components/pages/signUp/SignUpForm.tsx ================================================ import Link from 'next/link'; import { FC, useEffect, useState } from 'react'; import { useForm } from 'react-hook-form'; import { AppState } from '../../../store'; import classes from './SignUpForm.module.scss'; interface SignUpFormProps { onSignUp: Function; name: { firstName: string | null; lastName: string | null; }, signUpState: AppState["signUp"] } type FormData = { firstName: string; lastName: string; } const SignUpForm: FC = (props) => { const { name, onSignUp, signUpState } = props; const { register, handleSubmit, setValue, setError, formState: { errors } } = useForm(); const [ isLoading, setIsLoading] = useState(false); useEffect(() => { if(name.firstName && name.lastName) { setValue("firstName", name.firstName); setValue("lastName", name.lastName); } }, [name.firstName, name.lastName, setValue]) useEffect(() => { if(signUpState.errors) { signUpState.errors.firstName && setError("firstName", { type: 'custom', message: signUpState.errors.firstName }); signUpState.errors.lastName && setError("lastName", { type: 'custom', message: signUpState.errors.lastName }); } }, [signUpState]) const onSubmitHandler = (data: any) => { setIsLoading(true); onSignUp(data) .then(() => { setIsLoading(false); }) }; return (

Generate Account

{errors.firstName &&

{errors.firstName.message}

}
{errors.lastName &&

{errors.lastName.message}

}

Already have an account? Login

) } export default SignUpForm; ================================================ FILE: aquila/view/components/pages/signUp/SignUpPageWrapper.tsx ================================================ import { FC } from "react"; import { AppState } from '../../../store/'; import BoxCenterLayout from "../../layout/boxCenter/BoxCenterLayout"; import SecretKey from "./SecretKey"; import SignUpForm from "./SignUpForm"; interface SignUpPageWrapperProps { onSignUp: Function, name: { firstName: string | null; lastName: string | null; }, signUpState: AppState["signUp"]; accountCreated: boolean; } const SignUpPageWrapper: FC = (props) => { return ( { !props.accountCreated && } { props.accountCreated && } ) } export default SignUpPageWrapper; ================================================ FILE: aquila/view/components/pages/subscription/Collection.module.scss ================================================ .collection { display: flex; align-items: center; gap: 10px; border: 1px solid #EFF2F6; padding: 10px; border-radius: 3px; &__link { color: #202A38; text-decoration: none; transition: all .3s; } &__link:hover { color: #0FBD86; } } ================================================ FILE: aquila/view/components/pages/subscription/Collection.tsx ================================================ import Avatar from 'boring-avatars'; import Link from 'next/link'; import { FC } from 'react'; import classes from './Collection.module.scss'; interface CollectionProps { name: string; collectionId: string; } const Collection: FC = (props) => { const { name, collectionId } = props; return ( ) } export default Collection; ================================================ FILE: aquila/view/components/pages/subscription/HomePageWrapper.tsx ================================================ import { FC, useEffect, useState } from "react"; import { Oval } from 'react-loader-spinner'; import { AppState } from "../../../store"; import { Collection } from "../../../store/slices/types/Collection"; import { Customer } from "../../../store/slices/types/Customer"; import MainLayout from "../../layout/main/MainLayout" import Container from "../../ui/layout/Container"; import classes from "./HomePageWrapper.module.scss"; import SearchBar from "./SearchBar"; import SearchPageProfile from "./SearchPageProfile"; import SearchResults from "./SearchResults"; interface HomePageWrapperProps { customer: Customer | null; collection: Collection | null; bookmarksState: AppState["getLoggedInCustBookmarksByCollectionId"]; onSearch: Function; onClickNextPage: Function; onClickPrevPage: Function; } const HomePageWrapper: FC = (props) => { const { collection, customer, bookmarksState, onSearch, onClickNextPage, onClickPrevPage } = props; const [hasNext, setHasNext] = useState(false); const [hasPrev, setHasPrev] = useState(false); useEffect(() => { if(bookmarksState.currentPage && bookmarksState.totalPages) { if(bookmarksState.currentPage < bookmarksState.totalPages) { setHasNext(true); }else { setHasNext(false); } if(bookmarksState.currentPage > 1) { setHasPrev(true) }else { setHasPrev(false) } } }, [bookmarksState]) return (
{bookmarksState.status === "pending" && } {bookmarksState.bookmarks && }
{collection && customer && }
) } export default HomePageWrapper; ================================================ FILE: aquila/view/components/pages/subscription/SearchBar.module.scss ================================================ .search-bar { padding: 10px 5px; &__container { display: flex; border-bottom: 1px solid #AEBCD1; padding: 10px 5px; } &__input { margin-right: 10px; width: 100%; border: none; outline: none; } &__input::placeholder { color: #AEBCD1; } &__btn { border: none; outline: none; background: transparent; } } ================================================ FILE: aquila/view/components/pages/subscription/SearchBar.tsx ================================================ import { FC, useRef } from 'react'; import { IoSearch } from 'react-icons/io5'; import classes from './SearchBar.module.scss'; interface SearchBarProps { onSearch: Function } const SearchBar: FC = (props) => { const { onSearch } = props; const searchValueRef = useRef(null); const onSubmitHandler = (e: any) => { e.preventDefault(); onSearch(searchValueRef.current?.value) } return (
) } export default SearchBar; ================================================ FILE: aquila/view/components/pages/subscription/SearchPageProfile.module.scss ================================================ .search-profile { padding: 15px; margin-top: 20px; border: 1px solid #CED7E3; border-radius: 3px; &__header { display: flex; justify-content: space-between; margin-bottom: 10px; } &__header-subscribe-btn { margin-top: 5px; background: transparent; border: 1px solid #8DA2BE; outline: none; border-radius: 3px; color: #2E3D51; transition: all .3s; cursor: pointer; } &__header-subscribe-btn:hover { color: #0FBD86; border-color: #0FBD86; } &__header-left { display: flex; flex-direction: column; gap: 10px; } &__header-name { color: #202A38; font-size: 25px; } &__body-desc { color: #415572; margin: 0px; font-size: 14px; margin-bottom: 20px; } &__body-bookmark-title { font-size: 14px; margin-bottom: 5px; color: #415572; } &__body-share-bookmark-box { overflow: hidden; background: #EFF2F6; position: relative; box-sizing:border-box; margin: 0px auto; padding: 3px 4px; border-radius: 3px; } &__body-share-bookmark-btn { position: absolute; right: 0px; top: 0px; bottom: 0px; font-size: 12px; padding: 5px; color: #fff; background: #3ABFF8; border-radius: 3px; cursor: pointer; transition: all .3s; } &__body-share-bookmark-btn:hover { background: #089DDE; } &__body-share-bookmark-link { width: 100%; margin: 0px; border: none; outline: none; background: transparent; font-size: 12px; color: #415572; } &__body-feedback-container { margin-top: 20px; } &__body-feedback-txt { color: #097654; text-decoration: none; font-size: 14px; transition: all .3s; } &__body-feedback-txt:hover { color: #0FBD86; } } ================================================ FILE: aquila/view/components/pages/subscription/SearchPageProfile.tsx ================================================ import { VscFeedback } from 'react-icons/vsc'; import { toast } from 'react-toastify'; import Avatar from "boring-avatars"; import classes from './SearchPageProfile.module.scss'; import { Customer } from '../../../store/slices/types/Customer'; import { FC, useRef } from 'react'; import { Collection } from '../../../store/slices/types/Collection'; interface SearchPageProfileProps { customer: Customer; collection: Collection; } const SearchPageProfile: FC = ({ customer, collection }) => { const bookmarkShareLinkRef = useRef(null); const OnClickCopyBtnHandler = () => { navigator.clipboard.writeText(bookmarkShareLinkRef.current?.value || ""); toast("Collection link copied", { position: "top-center"}); } return (

{`${customer.firstName} ${customer.lastName}`}

{collection.desc}

Share this bookmark

Copy
) } export default SearchPageProfile; ================================================ FILE: aquila/view/components/pages/subscription/SearchResultItem.module.scss ================================================ .search-result-item { border-bottom: 1px solid #CED7E3; padding-bottom: 15px; margin-bottom: 10px; word-wrap: break-word; &__title { margin-bottom: 2px; font-size: 22px; } &__title-link { font-size: 22px; color: #202A38; transition: all .3s; cursor: pointer; text-decoration: none; } &__title-link:hover { color: #0FBD86; } &__meta-info { padding-bottom: 0px; margin-bottom: 0px; font-size: 12px; color: #8DA2BE; } &__site-link { font-size: 14px; color: #536E92; transition: all .3s; } &__site-link:hover { color: #9B5094; } &__site-desc { color: #415572; margin-top: 5px; margin-bottom: 3px; padding-bottom: 0px; } } ================================================ FILE: aquila/view/components/pages/subscription/SearchResultItem.tsx ================================================ import { FC } from 'react'; import moment from 'moment'; import classes from './SearchResultItem.module.scss'; interface SearchResultItemProps { title: string; url: string; summary: string; createdAt: string; } const SearchResultItem: FC = (props) => { const { title, url, summary, createdAt } = props; return (

{title}

Updated {moment(createdAt).fromNow()}

{summary.length > 255 ? summary.substring(0, 255)+'...' : summary}

{url}
); } export default SearchResultItem; ================================================ FILE: aquila/view/components/pages/subscription/SearchResults.module.scss ================================================ .search-result { &__results { margin-top: 20px; } &__item { margin-bottom: 25px; } &__header-info-desc { color: #536E92; } &__pagination { margin-top: 15px; display: flex; gap: 10px; } &__pagination-link { font-size: 14px; text-decoration: none; background: #EFF2F6; color: #202A38; padding: 5px; transition: all .3s; border-radius: 3px; } &__pagination-link:hover { background: #CED7E3; } } ================================================ FILE: aquila/view/components/pages/subscription/SearchResults.tsx ================================================ import { FC } from "react"; import { Bookmark } from "../../../store/slices/types/Bookmark"; import SearchResultItem from "./SearchResultItem"; import classes from './SearchResults.module.scss'; interface SearchResultsProps { bookmarks: Bookmark[]; hasNext: boolean; hasPrev: boolean; onClickNextPage: Function; onClickPrevPage: Function; totalRecords: number | null; } const SearchResults: FC = (props) => { const { bookmarks, hasNext, hasPrev, onClickNextPage, onClickPrevPage, totalRecords } = props; const onNextPageHandler = (e: any) => { e.preventDefault(); onClickNextPage() } const onPrevPageHandler = (e:any) => { e.preventDefault(); onClickPrevPage() } return (
{totalRecords !== null ?

Received {totalRecords} results

: null}
{bookmarks.map((bookmark, index) => (
))}
{hasNext && Previous} {hasPrev && Next}
); } export default SearchResults; ================================================ FILE: aquila/view/components/pages/subscription/SubscribedCollections.module.scss ================================================ .subscribed-collections { border: 1px solid #CED7E3; border-radius: 3px; &__header { padding: 20px; } &__header-title { font-size: 20px; color: #202A38; } &__body { padding: 20px; } &__list { list-style: none; display: flex; flex-direction: column; gap: 20px } } ================================================ FILE: aquila/view/components/pages/subscription/SubscribedCollections.tsx ================================================ import { FC } from "react"; import { Collection } from "../../../store/slices/types/Collection"; import { default as CollectionComponent } from "./Collection" import classes from './SubscribedCollections.module.scss'; interface SubscribedCollectionsProps { collections: Collection[] } const SubscribedCollections: FC = (props) => { const { collections } = props; return (

Subscriptions

    {collections.map((collection) => (
  • ))}
) } export default SubscribedCollections; ================================================ FILE: aquila/view/components/pages/subscription/SubscriptionPageWrapper.module.scss ================================================ .subscription-page { display: flex; gap: 50px; &__search-area { flex-basis: 900px; } &__search-bar { margin-top: 20px; width: 600px; } &__search-results { margin-top: 30px; } &__sidebar { padding-top: 20vh; flex: 1 1 0; } } @media only screen and (max-width: 992px) { .subscription-page { gap: 0px; flex-direction: column; &__search-area { flex-basis: auto; } &__search-bar { width: 80%; } &__sidebar { display: none; } } } ================================================ FILE: aquila/view/components/pages/subscription/SubscriptionPageWrapper.tsx ================================================ import { FC, useEffect, useState } from "react"; import { Oval } from 'react-loader-spinner'; import { AppState } from "../../../store"; import { Collection } from "../../../store/slices/types/Collection"; import { Customer } from "../../../store/slices/types/Customer"; import MainLayout from "../../layout/main/MainLayout" import Container from "../../ui/layout/Container"; import classes from "./SubscriptionPageWrapper.module.scss"; import SearchBar from "./SearchBar"; import SearchResults from "./SearchResults"; import SubscribedCollections from "./SubscribedCollections"; interface SubscriptionPageWrapperProps { subscribedCollections: Collection[] | null; bookmarksState: AppState["getCustomerSubscriptions"]; onSearch: Function; onClickNextPage: Function; onClickPrevPage: Function; } const SubscriptionPageWrapper: FC = (props) => { const { bookmarksState, onSearch, onClickNextPage, onClickPrevPage, subscribedCollections } = props; const [hasNext, setHasNext] = useState(false); const [hasPrev, setHasPrev] = useState(false); useEffect(() => { if(bookmarksState.currentPage && bookmarksState.totalPages) { if(bookmarksState.currentPage < bookmarksState.totalPages) { setHasNext(true); }else { setHasNext(false); } if(bookmarksState.currentPage > 1) { setHasPrev(true) }else { setHasPrev(false) } } }, [bookmarksState]) return (
{bookmarksState.status === "pending" && } {bookmarksState.bookmarks && }
{Array.isArray(subscribedCollections) && }
) } export default SubscriptionPageWrapper; ================================================ FILE: aquila/view/components/ui/alert/Alert.module.scss ================================================ .alert { padding: 10px; border-radius: 3px; &--success { background: #EAFAF4; color: #36D399; } &--info { background: #E6F7FE; color: #3ABFF8; } &--danger { background: #FEE7E7; color: #F87272; } &__text { font-size: 14px; text-align: center; } } ================================================ FILE: aquila/view/components/ui/alert/Alert.tsx ================================================ import { FC } from 'react'; import classes from './Alert.module.scss'; interface AlertProps { message: string; type: "danger" | "success" | "info"; } const alertClassMap = { danger: classes["alert--danger"], success: classes["alert--success"], info: classes["alert--info"] } const Alert: FC = ({ message, type}) => { return (

{message}

) } export default Alert; ================================================ FILE: aquila/view/components/ui/layout/Container.module.scss ================================================ .container { max-width: 1200px; margin: 0px auto; } @media only screen and (max-width: 1200px) { .container { padding: 0px 10px; } } ================================================ FILE: aquila/view/components/ui/layout/Container.tsx ================================================ import { FC } from "react"; import classes from './Container.module.scss'; interface ContainerProps { children: React.ReactNode; } const Container: FC = (props) => { return
{props.children}
} export default Container; ================================================ FILE: aquila/view/components/ui/modal/Modal.module.scss ================================================ .modal { width: 100%; height: 100%; position: fixed; top: 0px; left: 0px; background: rgb(32, 42, 56, 0.7); } ================================================ FILE: aquila/view/components/ui/modal/Modal.tsx ================================================ import { FC, useEffect, useState } from "react"; import ReactDOM from 'react-dom'; import classes from './Modal.module.scss'; interface ModelProps { children: React.ReactNode; onClose: Function } const Modal: FC = (props) => { const [mounted, setMounted] = useState(false); const onCloseHandler = (e:any) => { props.onClose(); } useEffect(() => { setMounted(true); }, []) return mounted ? ReactDOM.createPortal(
{props.children}
, document.getElementById("modal") as HTMLElement ): null; } export default Modal; ================================================ FILE: aquila/view/components/ui/progressLoader/ProgressLoader.module.scss ================================================ .progress-loader { position: fixed; top: 0px; height: 4px; display: block; width: 100%; background-color: rgb(15, 189, 134, 0.2); border-radius: 2px; overflow: hidden; z-index: 3000; &__item { background-color: #0FBD86; } &__item:before{ content: ''; position: absolute; background-color: inherit; top: 0; left: 0; bottom: 0; will-change: left, right; animation: progress 1.1s cubic-bezier(0.65, 0.815, 0.735, 0.395) infinite; } } @keyframes progress { 0% { left: -35%; right: 100%; } 60% { left: 100%; right: -90%; } 100% { left: 100%; right: -90%; } } ================================================ FILE: aquila/view/components/ui/progressLoader/ProgressLoader.tsx ================================================ import React, { FC, useContext, useState } from 'react'; import classes from './ProgressLoader.module.scss'; interface ProgressLoaderCtxValue { status: boolean, setLoader: (status: boolean) => void}; const ProgressLoaderCtx = React.createContext(undefined) export const useProgressLoader = () => { const context = useContext(ProgressLoaderCtx); if(!context) { throw new Error("useProgressLoader must use inside a ProgressLoaderProvider"); } return context; } interface ProgressLoaderProviderProps { children: React.ReactNode; } const ProgressLoader = () => { const { status } = useProgressLoader(); if (!status) return null; return (
) } export const ProgressLoaderProvider: FC = ({ children }) => { const [status, setLoader] = useState(false); return ( {children} ); } ================================================ FILE: aquila/view/middleware.ts ================================================ export { default } from 'next-auth/middleware'; export const config = { matcher: ['/home', '/profile', '/subscription', '/account/edit-profile'] } ================================================ FILE: aquila/view/next.config.js ================================================ /** @type {import('next').NextConfig} */ const nextConfig = { reactStrictMode: true, swcMinify: true, output: "standalone" } module.exports = nextConfig ================================================ FILE: aquila/view/package.json ================================================ { "name": "aquila-network-ui", "version": "0.1.0", "private": true, "scripts": { "dev": "next dev", "build": "next build", "start": "next start", "lint": "next lint" }, "dependencies": { "@reduxjs/toolkit": "^1.8.6", "axios": "^1.1.3", "boring-avatars": "^1.7.0", "jsonwebtoken": "^8.5.1", "moment": "^2.29.4", "next": "12.3.1", "next-auth": "^4.14.0", "next-redux-wrapper": "^8.0.0", "react": "18.2.0", "react-dom": "18.2.0", "react-hook-form": "^7.37.0", "react-icons": "^4.6.0", "react-loader-spinner": "^5.3.4", "react-redux": "^8.0.4", "react-toastify": "^9.0.8" }, "devDependencies": { "@types/jsonwebtoken": "^8.5.9", "@types/node": "18.11.0", "@types/react": "18.0.21", "@types/react-dom": "18.0.6", "eslint": "8.25.0", "eslint-config-next": "12.3.1", "sass": "^1.55.0", "typescript": "4.8.4" } } ================================================ FILE: aquila/view/pages/_app.tsx ================================================ import type { AppProps } from 'next/app'; import { Provider } from 'react-redux'; import { SessionProvider } from 'next-auth/react'; import { ToastContainer} from 'react-toastify'; import 'react-toastify/dist/ReactToastify.css'; import { wrapper } from '../store'; import '../styles/globals.scss'; import { Session } from 'next-auth'; import InitComponent from '../components/hoc/InitComponent'; import BaseLayout from '../components/layout/base/baseLayout'; interface ApplicationProps { session: Session } function MyApp({ Component, pageProps: { session, ...rest} }: AppProps) { const {store, props} = wrapper.useWrappedStore(rest); return ( ); } export default MyApp; ================================================ FILE: aquila/view/pages/_document.tsx ================================================ import { Head, Html, Main, NextScript } from "next/document"; export default function Document() { return (
) } ================================================ FILE: aquila/view/pages/account/edit-profile.tsx ================================================ import { signIn } from "next-auth/react"; import { useEffect } from "react"; import { toast, ToastOptions } from 'react-toastify'; import EditProfileWrapper from "../../components/pages/account/editProfile/EditProfileWrapper"; import { useProgressLoader } from "../../components/ui/progressLoader/ProgressLoader"; import { useAppDispatch, useAppSelector } from "../../store"; import { selectAuth } from "../../store/slices/auth"; import { activateCustomer, selectActivateCustomer } from "../../store/slices/customer/activateCustomer"; import { getCurrentLoggedInCustomer, selectGetCurrentLoggedInCustomer } from "../../store/slices/customer/getCurrentLoggedInCustomer"; import { selectUpdateCustomer, updateCustomer, UpdateCustomerData } from "../../store/slices/customer/updateCustomer"; const EditProfilePage = () => { const dispatch = useAppDispatch(); const currentLoggedInCustomer = useAppSelector(selectGetCurrentLoggedInCustomer); const authState = useAppSelector(selectAuth); const updateCustomerState = useAppSelector(selectUpdateCustomer); const activateCustomerState = useAppSelector(selectActivateCustomer); const { setLoader } = useProgressLoader(); useEffect(() => { dispatch(getCurrentLoggedInCustomer()) }, []) const updateFormSubmissionHandler = async (data: UpdateCustomerData) => { const toastOptions: ToastOptions = { position: "top-center", hideProgressBar: true, } setLoader(true); if(authState.accountStatus === "TEMPORARY") { try { const customer = await dispatch(activateCustomer(data)).unwrap() setLoader(false); const resp = await signIn('credentials',{ redirect: false, secretKey: customer.secretKey }); if(resp?.ok) { toast("Account Activated Successfully", { ...toastOptions, type: "success"}) return true; } }catch(e) { if(e instanceof Error) { toast(e.message, { ...toastOptions, type: "error"}); } setLoader(false); return false; } }else { try { await dispatch(updateCustomer(data)).unwrap() toast("Profile Updated successfully", { ...toastOptions, type: "success"}) setLoader(false); return true; }catch(e) { if(e instanceof Error) { toast(e.message, { ...toastOptions, type: "error"}); } setLoader(false); return false; } } } return ( ) } export default EditProfilePage; ================================================ FILE: aquila/view/pages/api/auth/[...nextauth].ts ================================================ import NextAuth, { NextAuthOptions, User } from 'next-auth'; import Credentials from 'next-auth/providers/credentials'; import jwt from 'jsonwebtoken'; import api from '../../../utils/api'; interface AuthPayload { accountStatus: string; customerId: string; firstName: string; lastName: string; createdAt: string; } export const authOptions: NextAuthOptions = { providers: [ Credentials({ name: 'credentials', credentials: { secretKey: {}}, async authorize(credentials){ let token = ''; let response; try{ response = await api.post('/auth/login', { secretKey: credentials?.secretKey }); console.log(response.data); token = response.data?.token; }catch(e: unknown) { throw new Error(response?.data.message || "Something went wrong"); } if(!token) { throw new Error('Something went wrong'); } const data = jwt.decode(token) as unknown as AuthPayload; return { customerId: data.customerId, firstName: data.firstName, lastName: data.lastName, createdAt: data.createdAt, accountStatus: data.accountStatus, token: token } as User; } }) ], callbacks: { session({ session, token}) { if(token.customerId) { session.user.customerId = token.customerId; } if(token.firstName) { session.user.firstName = token.firstName; } if(token.lastName) { session.user.lastName = token.lastName; } if(token.accountStatus) { session.user.createdAt = token.createdAt; } if(token.accountStatus) { session.user.accountStatus = token.accountStatus; } if(token.token) { session.user.token = token.token; } return session; }, jwt({ token, user}) { if(user) { token.customerId = user.customerId; token.firstName = user.firstName; token.lastName = user.lastName; token.createdAt = user.createdAt; token.accountStatus = user.accountStatus; token.token = user.token; } return token; } }, pages: { signIn: '/sign-in' }, session: { strategy: "jwt", maxAge: 60 * 60 * 2, // 2 hours } }; export default NextAuth(authOptions) ================================================ FILE: aquila/view/pages/api/hello.ts ================================================ // Next.js API route support: https://nextjs.org/docs/api-routes/introduction import type { NextApiRequest, NextApiResponse } from 'next' type Data = { name: string } export default function handler( req: NextApiRequest, res: NextApiResponse ) { res.status(200).json({ name: 'John Doe' }) } ================================================ FILE: aquila/view/pages/collection/bookmark/[collectionId].tsx ================================================ import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { toast } from 'react-toastify'; import CollectionBookmarksPageWrapper from "../../../components/pages/collection/bookmark/CollectionBookmarks/CollectionBookmarksPageWrapper"; import { useProgressLoader } from "../../../components/ui/progressLoader/ProgressLoader"; import { useAppDispatch, useAppSelector } from "../../../store"; import { selectAuth } from "../../../store/slices/auth"; import {getPublicBookmarksByCollectionId, GetPublicBookmarksByCollectionIdInputOptions, selectGetPublicBookmarksByCollectionId } from "../../../store/slices/bookmark/getPublicBookmarksByCollectionId"; import { getCollectionById, selectGetCollectionById } from "../../../store/slices/collection/getCollectionById"; import { isCollectionSubscribed, selectIsCollectionSubscribed } from "../../../store/slices/collection/isCollectionSubscribed"; import { selectSubscribeCollectionById, subscribeCollectionById } from "../../../store/slices/collection/subscribeCollectionById"; import { selectUnSubscribeCollectionById, unSubscribeCollectionById, unSubscribeCollectionByIdSlice } from "../../../store/slices/collection/unSubscribeCollectionById"; import { getCustomerById, selectGetCustomerById } from "../../../store/slices/customer/getCustomerById"; const ViewCollectionBookmarks = () => { const router = useRouter(); const { collectionId } = router.query as { collectionId: string}; const authState = useAppSelector(selectAuth); // const currentLoggedInCustomerState = useAppSelector(selectGetCurrentLoggedInCustomer); const getCustomerByIdState = useAppSelector(selectGetCustomerById); const getPublicCollectionBookmarksState = useAppSelector(selectGetPublicBookmarksByCollectionId); const getCollectionByIdState = useAppSelector(selectGetCollectionById); const isCollectionSubscribedState = useAppSelector(selectIsCollectionSubscribed); const subscribeCollectionByIdState = useAppSelector(selectSubscribeCollectionById); const unSubscribeCollectionByIdState = useAppSelector(selectUnSubscribeCollectionById); const dispatch = useAppDispatch(); const [query, setQuery] = useState(null); const [currentPage, setCurrentPage] = useState(1); const { setLoader } = useProgressLoader(); useEffect(() => { if(authState.isSignedIn) { dispatch((isCollectionSubscribed(collectionId))) } }, [dispatch, authState.isSignedIn, collectionId, subscribeCollectionByIdState.collectionSubscription, unSubscribeCollectionByIdState.collectionSubscription]) useEffect(() => { if(collectionId) { dispatch(getCollectionById(collectionId)); } }, [dispatch, collectionId]) useEffect(() => { if(getCollectionByIdState.collection) { dispatch(getCustomerById(getCollectionByIdState.collection.customerId)) } }, [dispatch, getCollectionByIdState.collection]) useEffect(() => { const options: GetPublicBookmarksByCollectionIdInputOptions = { collectionId: collectionId, page: currentPage } if(query) { options.query = query; } dispatch(getPublicBookmarksByCollectionId(options)) }, [dispatch, query, currentPage, collectionId]) const onSearchHandler = (data: string) => { if(data) { setQuery(data); setCurrentPage(1); }else { setQuery(null); } } const onClickNextPageHandler = () => { setCurrentPage(currentPage + 1) } const onClickPrevPageHandler = () => { setCurrentPage(currentPage - 1) } const onSubscribeHandler = async (collectionId: string) => { if(!authState.isSignedIn) { router.push('/sign-in'); return; } try { setLoader(true); const resp = await dispatch(subscribeCollectionById(collectionId)).unwrap(); toast("Collection Subscribed Successfully", { position: "top-center"}); setLoader(false); return resp; }catch(e: any) { setLoader(false); toast(e.message || "Something went wrong!", { position: "top-center", type: "error"}); return false; } } const onUnSubscribeHandler = async (collectionId: string) => { try { setLoader(true); const resp = await dispatch(unSubscribeCollectionById(collectionId)).unwrap(); toast("Collection Unsubscribed Successfully", { position: "top-center"}); setLoader(false); return resp; }catch(e: any) { setLoader(false); let message = e.message || "Something went wrong!"; toast(message, { position: "top-center"}); return false; } } return } export default ViewCollectionBookmarks; ================================================ FILE: aquila/view/pages/explore.tsx ================================================ import { FC } from "react"; import { useSelector } from "react-redux"; import ExplorePageWrapper from "../components/pages/explore/ExplorePageWrapper"; import { Collection } from "../store/slices/types/Collection"; import { wrapper } from '../store/index'; import api from "../utils/api"; import { selectGetFeaturedCollections, setGetFeaturedCollections } from '../store/slices/collection/getFeaturedCollections'; import { selectGetAllPublicCollections, setGetAllPublicCollections } from "../store/slices/collection/getAllPublicCollections"; interface ExplorePageProps { featuredCollections: { totalRecords: number; totalPages: number; limit: number; collections: Collection[] } } const ExplorePage: FC = (props) => { const featuredCollectionsSate = useSelector(selectGetFeaturedCollections); const publicCollectionsState = useSelector(selectGetAllPublicCollections); return ( ) } export const getStaticProps = wrapper.getStaticProps((store) => async () => { const response = await api.get('/collection/public/featured'); const featuredCollections = response.data; const publicCollectionsResp = await api.get('/collection/public'); const publicCollections = publicCollectionsResp.data; store.dispatch(setGetFeaturedCollections(featuredCollections)) store.dispatch(setGetAllPublicCollections(publicCollections)) return { props: {}, revalidate: 600 } }) export default ExplorePage; ================================================ FILE: aquila/view/pages/home.tsx ================================================ import { useEffect, useState } from "react"; import { toast } from "react-toastify"; import HomePageWrapper from "../components/pages/home/HomePageWrapper"; import { useProgressLoader } from "../components/ui/progressLoader/ProgressLoader"; import { useAppSelector, useAppDispatch } from "../store"; import {getLoggedInCustBookmarksByCollectionId, GetLoggedInCustBookmarksByCollectionIdInputOptions, selectGetLoggedInCustBookmarksByCollectionId } from "../store/slices/bookmark/getLoggedInCustBookmarksByCollectionId"; import { selectGetLoggedInCustCollections } from "../store/slices/collection/getLoggedInCustCollections"; import { isCollectionSubscribed, selectIsCollectionSubscribed } from "../store/slices/collection/isCollectionSubscribed"; import { selectSubscribeCollectionById, subscribeCollectionById } from "../store/slices/collection/subscribeCollectionById"; import { selectUnSubscribeCollectionById, unSubscribeCollectionById } from "../store/slices/collection/unSubscribeCollectionById"; import { getCurrentLoggedInCustomer, selectGetCurrentLoggedInCustomer } from "../store/slices/customer/getCurrentLoggedInCustomer"; const HomePage = () => { const currentLoggedInCustomerState = useAppSelector(selectGetCurrentLoggedInCustomer); const getLoggedInCustCollections = useAppSelector(selectGetLoggedInCustCollections) const getLoggedInCustBookmarksState = useAppSelector(selectGetLoggedInCustBookmarksByCollectionId); const isCollectionSubscribedState = useAppSelector(selectIsCollectionSubscribed); const subscribeCollectionByIdState = useAppSelector(selectSubscribeCollectionById); const unSubscribeCollectionByIdState = useAppSelector(selectUnSubscribeCollectionById); const dispatch = useAppDispatch(); const [query, setQuery] = useState(null); const [currentPage, setCurrentPage] = useState(1); const { setLoader } = useProgressLoader(); useEffect(() => { dispatch(getCurrentLoggedInCustomer()) }, [dispatch]) useEffect(() => { if(getLoggedInCustCollections.collecitons) { dispatch((isCollectionSubscribed( getLoggedInCustCollections.collecitons[0].id))) } }, [dispatch, getLoggedInCustCollections.collecitons, subscribeCollectionByIdState.collectionSubscription, unSubscribeCollectionByIdState.collectionSubscription]) useEffect(() => { if(getLoggedInCustCollections.collecitons && getLoggedInCustCollections.collecitons.length > 0) { const options: GetLoggedInCustBookmarksByCollectionIdInputOptions = { collectionId: getLoggedInCustCollections.collecitons[0].id, page: currentPage } if(query) { options.query = query; } dispatch(getLoggedInCustBookmarksByCollectionId(options)) } }, [dispatch, getLoggedInCustCollections.collecitons, query, currentPage]) const onSearchHandler = (data: string) => { if(data) { setQuery(data); setCurrentPage(1); }else { setQuery(null); } } const onClickNextPageHandler = () => { setCurrentPage(currentPage + 1) } const onClickPrevPageHandler = () => { setCurrentPage(currentPage - 1) } const onSubscribeHandler = async (collectionId: string) => { try { setLoader(true); const resp = await dispatch(subscribeCollectionById(collectionId)).unwrap(); toast("Collection Subscribed Successfully", { position: "top-center"}); setLoader(false); return resp; }catch(e: any) { setLoader(false); toast(e.message || "Something went wrong!", { position: "top-center", type: "error"}); return false; } } const onUnSubscribeHandler = async (collectionId: string) => { try { setLoader(true); const resp = await dispatch(unSubscribeCollectionById(collectionId)).unwrap(); toast("Collection Unsubscribed Successfully", { position: "top-center"}); setLoader(false); return resp; }catch(e: any) { setLoader(false); let message = e.message || "Something went wrong!"; toast(message, { position: "top-center"}); return false; } } return ( 0 ? getLoggedInCustCollections.collecitons[0] : null} customer={currentLoggedInCustomerState.customer} bookmarksState={getLoggedInCustBookmarksState} onSubscribe={onSubscribeHandler} onUnsubscribe={onUnSubscribeHandler} isCollectionSubscribed={isCollectionSubscribedState.isSubscribed} onClickNextPage={onClickNextPageHandler} onClickPrevPage={onClickPrevPageHandler} onSearch={onSearchHandler} /> ); } // export const getServerSideProps = wrapper.getServerSideProps(store=> async (ctx) => { // const session = await unstable_getServerSession(ctx.req, ctx.res, authOptions); // if(session) { // store.dispatch(signIn({ token: session?.user.token, accountStatus: session?.user.accountStatus, customer: { // firstName: session?.user.firstName, // lastName: session?.user.lastName, // customerId: session?.user.customerId // }})) // } // return {props: {}}; // }); export default HomePage; ================================================ FILE: aquila/view/pages/index.tsx ================================================ import type { NextPage } from 'next' import IndexPageWrapper from '../components/pages/index/IndexPageWrapper'; const IndexPage: NextPage = () => { return ( ); } export default IndexPage; ================================================ FILE: aquila/view/pages/sign-in.tsx ================================================ import { NextPage } from "next"; import SignInPageWrapper from "../components/pages/signIn/SignInPageWrapper"; import { signIn } from 'next-auth/react'; import { useRouter } from "next/router"; import { useEffect } from "react"; import { toast } from "react-toastify"; import { useAppSelector } from "../store"; import { selectAuth } from "../store/slices/auth"; import { useProgressLoader } from "../components/ui/progressLoader/ProgressLoader"; const SignInPage: NextPage = () => { const router = useRouter(); const authState = useAppSelector(selectAuth); const { setLoader } = useProgressLoader(); useEffect(() => { if(authState.isSignedIn) { router.push('/home'); } },[router, authState]); const signInHandler = async (secretKey: string) => { setLoader(true); const resp = await signIn('credentials',{ redirect: false, secretKey }); if(resp?.ok) { router.push('/home'); setLoader(false); return; } toast("Invalid credentials", { type: "error", position: "top-center"}); setLoader(false); } return } export default SignInPage; ================================================ FILE: aquila/view/pages/sign-up.tsx ================================================ import { NextPage } from "next"; import { signIn } from "next-auth/react"; import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { toast, ToastOptions } from 'react-toastify'; import SignUpPageWrapper from "../components/pages/signUp/SignUpPageWrapper"; import { useProgressLoader } from "../components/ui/progressLoader/ProgressLoader"; import { useAppDispatch, useAppSelector } from "../store"; import { selectAuth } from "../store/slices/auth"; import { fetchNames, removeGeneratedNames, selectGeneratedName } from "../store/slices/generateName"; import { selectSignUp, signUp } from "../store/slices/signup"; const SignUpPage: NextPage = () => { const dispatch = useAppDispatch(); const router = useRouter(); const authState = useAppSelector(selectAuth); const signUpState = useAppSelector(selectSignUp); const [accountCreated, setAccountCreated] = useState(false); const { setLoader } = useProgressLoader(); useEffect(() => { if(authState.isSignedIn && !accountCreated) { router.replace('/home'); } }, [router, authState]) useEffect(() => { dispatch(fetchNames()); return () => { dispatch(removeGeneratedNames()); } }, [dispatch]) const generatedName = useAppSelector(selectGeneratedName); const signUpHandler = async (data: any) => { const toastOptions: ToastOptions = { position: "top-center", hideProgressBar: true, } setLoader(true); try{ const signUpResult = await dispatch(signUp(data)).unwrap() setLoader(false); toast("Account Genreated", { ...toastOptions, type: "success"}) const result = await signIn("credentials", {secretKey: signUpResult.secretKey, redirect: false}); debugger; if(result?.ok) { setAccountCreated(true); } return true; }catch(e: any) { debugger; setLoader(false); toast(e.message, { ...toastOptions, type: "error"}); return false; } } return ( ); } export default SignUpPage; ================================================ FILE: aquila/view/pages/subscription.tsx ================================================ import { useEffect, useState } from "react"; import SubscriptionPageWrapper from "../components/pages/subscription/SubscriptionPageWrapper"; import { useAppSelector, useAppDispatch } from "../store"; import { getCustomerSubscriptions, GetCustomerSubscriptionsInputOptions, selectGetCustomerSubscriptions } from "../store/slices/collection/getCustomerSubscriptions"; import { getSubscribedCollections, selectGetSubscribedCollections } from "../store/slices/collection/getSubscribedCollections"; const SubscriptionPage = () => { const getCustomerSubscriptionsState = useAppSelector(selectGetCustomerSubscriptions); const getSubscribedCollectionsState = useAppSelector(selectGetSubscribedCollections); const dispatch = useAppDispatch(); const [query, setQuery] = useState(null); const [currentPage, setCurrentPage] = useState(1); useEffect(() => { dispatch(getSubscribedCollections()) }, [dispatch]) useEffect(() => { const options: GetCustomerSubscriptionsInputOptions = { page: currentPage } if(query) { options.query = query; } dispatch(getCustomerSubscriptions(options)) }, [dispatch, query, currentPage]) const onSearchHandler = (data: string) => { if(data) { setQuery(data); setCurrentPage(1); }else { setQuery(null); } } const onClickNextPageHandler = () => { setCurrentPage(currentPage + 1) } const onClickPrevPageHandler = () => { setCurrentPage(currentPage - 1) } return ( ) } export default SubscriptionPage; ================================================ FILE: aquila/view/pages/test.tsx ================================================ const TestPage = () => { return

Protected

; } export default TestPage; ================================================ FILE: aquila/view/store/index.ts ================================================ import { Action, configureStore, ThunkAction } from "@reduxjs/toolkit" import { createWrapper } from "next-redux-wrapper"; import { TypedUseSelectorHook, useDispatch, useSelector } from "react-redux"; import authReducer from './slices/auth'; import getCurrentLoggedInCustomerReducer from "./slices/customer/getCurrentLoggedInCustomer"; import generatedNameReducer from './slices/generateName'; import signUpReducer from './slices/signup'; import addLinkReducer from './slices/bookmark/addLink'; import getLoggedInCustCollectionsReducer from './slices/collection/getLoggedInCustCollections'; import activateCustomerReducer from './slices/customer/activateCustomer'; import updateCustomerReducer from './slices/customer/updateCustomer'; import getFeaturedCollectionsReducer from "./slices/collection/getFeaturedCollections"; import getLoggedInCustBookmarksByCollectionIdReducer from "./slices/bookmark/getLoggedInCustBookmarksByCollectionId"; import getPublicBookmarksByCollectionIdReducer from "./slices/bookmark/getPublicBookmarksByCollectionId"; import getCustomerByIdReducer from './slices/customer/getCustomerById'; import getCollectionByIdReducer from './slices/collection/getCollectionById'; import subscribeCollectionByIdReducer from './slices/collection/subscribeCollectionById'; import isCollectionSubscribedReducer from './slices/collection/isCollectionSubscribed'; import unSubscribbeCollectionByIdReducer from './slices/collection/unSubscribeCollectionById'; import getAllPublicCollectionsReducer from "./slices/collection/getAllPublicCollections"; import getCustomerSubscriptionsReducer from './slices/collection/getCustomerSubscriptions'; import getSubscribedCollectionsReducer from './slices/collection/getSubscribedCollections'; export const createStore = () => { return configureStore({ reducer: { auth: authReducer, generatedName: generatedNameReducer, signUp: signUpReducer, getCurrentLoggedInCustomer: getCurrentLoggedInCustomerReducer, addLink: addLinkReducer, getLoggedInCustCollections: getLoggedInCustCollectionsReducer, activateCustomer: activateCustomerReducer, updateCustomer: updateCustomerReducer, getFeaturedCollections: getFeaturedCollectionsReducer, getLoggedInCustBookmarksByCollectionId: getLoggedInCustBookmarksByCollectionIdReducer, getPublicBookmarksByCollectionId: getPublicBookmarksByCollectionIdReducer, getCustomerById: getCustomerByIdReducer, getCollectionById: getCollectionByIdReducer, subscribeCollectionById: subscribeCollectionByIdReducer, isCollectionSubscribed: isCollectionSubscribedReducer, unSubscribeCollectionById: unSubscribbeCollectionByIdReducer, getAllPublicCollections: getAllPublicCollectionsReducer, getCustomerSubscriptions: getCustomerSubscriptionsReducer, getSubscribedCollections: getSubscribedCollectionsReducer, }, middleware: (getDefaultMiddleware) => getDefaultMiddleware({ serializableCheck: false }) }); } export const store = createStore(); export type AppStore = ReturnType; export type AppState = ReturnType; export type AppDispatch = typeof store.dispatch; export type AppThunk = ThunkAction> export const useAppDispatch = () => useDispatch() export const useAppSelector: TypedUseSelectorHook = useSelector; export const wrapper = createWrapper(createStore); ================================================ FILE: aquila/view/store/slices/auth.ts ================================================ import { createSlice, PayloadAction } from "@reduxjs/toolkit"; import { HYDRATE } from "next-redux-wrapper"; import { AppState } from ".."; export interface AuthState { isSignedIn: boolean | null; token: string | null; accountStatus: string | null; customer: { customerId: string; firstName: string; lastName: string; createdAt: string; } | null; status: 'idle' | 'pending' | 'succeeded' | 'failed'; } interface SignInPayloadAction { token: string; accountStatus: string; customer: { customerId: string; firstName: string; lastName: string; createdAt: string; } } const initialState: AuthState = { isSignedIn: null, token: null, accountStatus: null, customer: null, status: 'idle' } export const authSlice = createSlice({ name: 'auth', initialState, reducers: { signIn: (state, action: PayloadAction) => { state.isSignedIn = true; state.customer = action.payload.customer; state.token = action.payload.token; state.accountStatus = action.payload.accountStatus; state.status = "succeeded"; }, signOut: (state) => { state.isSignedIn = false state.token = null; state.customer = null; state.accountStatus = null; state.status = "succeeded"; } }, extraReducers: { [HYDRATE]: (state, action) => { if(action.payload.auth.status !== "idle") { return { ...state, ...action.payload.auth } } } } }); export const { signIn, signOut} = authSlice.actions; export const selectAuth = (state: AppState) => state.auth export default authSlice.reducer; ================================================ FILE: aquila/view/store/slices/bookmark/addLink.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AppState } from "../.."; import api from "../../../utils/api"; import { AsyncThunkSubmissionError } from "../errors/AsyncThunkSubmissionError"; import { Bookmark } from "../types/Bookmark"; import { ValidationErrors } from "../types/validationErrors"; import { createSubmissionErrorFromErrObj } from "../utils/createError"; export interface AddLinkData { url: string; collectionId: string; } type AddLinkReqPayload = AddLinkData; type AddLinkResPayload = Bookmark; type AddLinkValidationErrors = ValidationErrors; interface AddLinkState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; bookmark: null | Bookmark; errorMessage: null |string; errors: AddLinkValidationErrors | null; } const initialState: AddLinkState = { status: 'idle', bookmark: null, errorMessage: null, errors: null } export const addLink = createAsyncThunk}>('/bookmark/add-link', async (data, thunkApi) => { try { const res = await api.post('/bookmark', data); return res.data; } catch(e) { let err: AsyncThunkSubmissionError= new AsyncThunkSubmissionError("Something went wrong", null); if(e instanceof Error) { err = createSubmissionErrorFromErrObj(e); } return thunkApi.rejectWithValue(err); } }); export const addLinkSlice = createSlice({ name: 'addLink', initialState, reducers: {}, extraReducers: async (builder) => { builder.addCase(addLink.pending, (state, action) => { state.bookmark = null; state.errorMessage = null; state.errors = null; state.status = "pending"; }) builder.addCase(addLink.fulfilled, (state, action) => { state.bookmark = action.payload; state.status = "succeeded"; state.errorMessage = null; state.errors = null; }) builder.addCase(addLink.rejected, (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || null; state.bookmark = null; if(action.payload) { state.errorMessage = action.payload.message; state.errors = action.payload.validationErrors; } }) } }) export const selectAddLink = (state: AppState) => state.addLink; export default addLinkSlice.reducer; ================================================ FILE: aquila/view/store/slices/bookmark/getLoggedInCustBookmarksByCollectionId.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Bookmark } from "../types/Bookmark"; interface GetLoggedInCustBookmarksByCollectionIdState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; bookmarks: null | Bookmark[]; totalPages: number | null; totalRecords: number | null; currentPage: number | null; limit: number | null; query: string | null; errorMessage: null | string; } interface GetLoggedInCustBookmarksByCollectionIdData { bookmarks: null | Bookmark[]; totalPages: number | null; totalRecords: number | null; currentPage: number | null; limit: number | null; query: string | null; } type GetLoggedInCustBookmarksByCollectionIdResPayload = Omit; export interface GetLoggedInCustBookmarksByCollectionIdInputOptions { collectionId: string; limit?: number; query?: string; page?: number; } export const getLoggedInCustBookmarksByCollectionId = createAsyncThunk('/bookmarks/collection/search', async (options: GetLoggedInCustBookmarksByCollectionIdInputOptions) => { try{ const resp = await api.get(`/bookmark/${options.collectionId}/search`, {params: options}); return { ...resp.data, query: options.query || null } }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) const initialState: GetLoggedInCustBookmarksByCollectionIdState = { status: 'idle', bookmarks: null, totalPages: null, totalRecords: null, currentPage: null, limit: null, query: null, errorMessage: null } const getLoggedInCustBookmarksByCollectionIdSlice = createSlice({ name: 'getBookmarksByCollectionId', initialState, reducers: {}, extraReducers: (builder) => { builder.addCase(getLoggedInCustBookmarksByCollectionId.pending, (state) => { state.status = 'pending'; state.bookmarks = null; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.query = null; state.errorMessage = null; }); builder.addCase(getLoggedInCustBookmarksByCollectionId.fulfilled, (state, action) => { state.status = 'succeeded'; state.bookmarks = action.payload.bookmarks; state.totalRecords = action.payload.totalRecords; state.totalPages = action.payload.totalPages; state.limit = action.payload.limit; state.query = action.payload.query; state.currentPage = action.payload.currentPage; state.errorMessage = null; }); builder.addCase(getLoggedInCustBookmarksByCollectionId.rejected, (state, action) => { state.status = 'failed'; state.bookmarks = null; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.query = null; state.errorMessage = action.error.message || "Something went wrong"; }); } }) export const selectGetLoggedInCustBookmarksByCollectionId = (state: AppState) => state.getLoggedInCustBookmarksByCollectionId; export default getLoggedInCustBookmarksByCollectionIdSlice.reducer; ================================================ FILE: aquila/view/store/slices/bookmark/getPublicBookmarksByCollectionId.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Bookmark } from "../types/Bookmark"; interface GetPublicBookmarksByCollectionIdState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; bookmarks: null | Bookmark[]; totalPages: number | null; totalRecords: number | null; currentPage: number | null; limit: number | null; query: string | null; errorMessage: null | string; } interface GetPublicBookmarksByCollectionIdData { bookmarks: null | Bookmark[]; totalPages: number | null; totalRecords: number | null; currentPage: number | null; limit: number | null; query: string | null; } type GetPublicBookmarksByCollectionIdResPayload = Omit; export interface GetPublicBookmarksByCollectionIdInputOptions { collectionId: string; limit?: number; query?: string; page?: number; } export const getPublicBookmarksByCollectionId = createAsyncThunk('/bookmarks/public/collection/search', async (options: GetPublicBookmarksByCollectionIdInputOptions) => { try{ const resp = await api.get(`/bookmark/public/${options.collectionId}/search`, {params: options}); return { ...resp.data, query: options.query || null } }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) const initialState: GetPublicBookmarksByCollectionIdState = { status: 'idle', bookmarks: null, totalPages: null, totalRecords: null, currentPage: null, limit: null, query: null, errorMessage: null } const getPublicBookmarksByCollectionIdSlice = createSlice({ name: 'getBookmarksByCollectionId', initialState, reducers: {}, extraReducers: (builder) => { builder.addCase(getPublicBookmarksByCollectionId.pending, (state) => { state.status = 'pending'; state.bookmarks = null; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.query = null; state.errorMessage = null; }); builder.addCase(getPublicBookmarksByCollectionId.fulfilled, (state, action) => { state.status = 'succeeded'; state.bookmarks = action.payload.bookmarks; state.totalRecords = action.payload.totalRecords; state.totalPages = action.payload.totalPages; state.limit = action.payload.limit; state.query = action.payload.query; state.currentPage = action.payload.currentPage; state.errorMessage = null; }); builder.addCase(getPublicBookmarksByCollectionId.rejected, (state, action) => { state.status = 'failed'; state.bookmarks = null; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.query = null; state.errorMessage = action.error.message || "Something went wrong"; }); } }) export const selectGetPublicBookmarksByCollectionId = (state: AppState) => state.getPublicBookmarksByCollectionId; export default getPublicBookmarksByCollectionIdSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/getAllPublicCollections.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { HYDRATE } from "next-redux-wrapper"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Collection } from "../types/Collection"; interface GetAllPublicCollectionsState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; collections: null | Collection[]; totalPages: number | null; totalRecords: number | null; currentPage: number | null; limit: number | null; errorMessage: null | string; } interface GetAllPublicCollectionsData { collections: Collection[]; totalPages: number; totalRecords: number; currentPage: number; limit: number } type GetAllPublicCollectionsResPayload = GetAllPublicCollectionsData; export const getAllPublicCollections = createAsyncThunk('/collection/public/all', async () => { try{ const resp = await api.get('/collection/'); return resp.data; }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) const initialState: GetAllPublicCollectionsState = { status: 'idle', collections: null, totalPages: null, totalRecords: null, currentPage: null, limit: null, errorMessage: null } const getAllPublicCollectionsSlice = createSlice({ name: 'getAllPublicCollections', initialState, reducers: { setGetAllPublicCollections: (state, action) => { state.status = 'succeeded'; state.collections = action.payload.collections; state.totalPages = action.payload.totalPages; state.totalRecords = action.payload.totalRecords; state.currentPage = action.payload.currentPage; state.limit = action.payload.currentPage; state.errorMessage = null; } }, extraReducers: { [HYDRATE]: (state, action) => { if(action.payload.status !== 'idle') { state.status = 'succeeded'; state.collections = action.payload.getAllPublicCollections.collections; state.totalPages = action.payload.getAllPublicCollections.totalPages; state.totalRecords = action.payload.getAllPublicCollections.totalRecords; state.currentPage = action.payload.getAllPublicCollections.currentPage; state.limit = action.payload.getAllPublicCollections.limit; return state; } }, [getAllPublicCollections.pending as unknown as string]: (state) => { state.status = 'pending'; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.collections = null; state.errorMessage = null; return state; }, [getAllPublicCollections.fulfilled as unknown as string]: (state, action) => { state.status = 'succeeded'; state.collections = action.payload.collections; state.totalPages = action.payload.totalPages; state.totalRecords = action.payload.totalRecords; state.currentPage = action.payload.currentPage; state.limit = action.payload.limit; state.errorMessage = null; return state; }, [getAllPublicCollections.rejected as unknown as string]: (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || "Something went wrong"; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.collections = null; return state; } } }) export const selectGetAllPublicCollections = (state: AppState) => state.getAllPublicCollections; export const { setGetAllPublicCollections } = getAllPublicCollectionsSlice.actions; export default getAllPublicCollectionsSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/getCollectionById.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Collection } from "../types/Collection"; interface GetCollectionByIdState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; collection: null | Collection; errorMessage: null |string; } const initialState: GetCollectionByIdState = { status: 'idle', collection: null, errorMessage: null } export const getCollectionById = createAsyncThunk('/collection/public/collectionId', async (collectionId: string) => { try{ const resp = await api.get(`/collection/public/${collectionId}`); return resp.data; }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) export const getCollectionByIdSlice = createSlice({ name: 'getCollectionByid', initialState, reducers: {}, extraReducers: (builder) => { builder.addCase(getCollectionById.pending, (state) => { state.status = 'pending'; state.collection = null; state.errorMessage = null; }); builder.addCase(getCollectionById.fulfilled, (state, action) => { state.collection = action.payload; state.status = "succeeded"; state.errorMessage = null; }); builder.addCase(getCollectionById.rejected, (state, action) => { state.status = "failed", state.errorMessage = action.error.message || "Something went wrong", state.collection = null; }) } }); export const selectGetCollectionById = (state: AppState) => state.getCollectionById; export default getCollectionByIdSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/getCustomerSubscriptions.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Bookmark } from "../types/Bookmark"; interface GetCustomerSubscriptionsState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; bookmarks: null | Bookmark[]; totalPages: number | null; totalRecords: number | null; currentPage: number | null; limit: number | null; query: string | null; errorMessage: null | string; } interface GetCustomerSubscriptionsData { bookmarks: null | Bookmark[]; totalPages: number | null; totalRecords: number | null; currentPage: number | null; limit: number | null; query: string | null; } type GetCustomerSubscriptionsResPayload = Omit; export interface GetCustomerSubscriptionsInputOptions { limit?: number; query?: string; page?: number; } export const getCustomerSubscriptions = createAsyncThunk('/subscription', async (options: GetCustomerSubscriptionsInputOptions) => { try{ const resp = await api.get(`/subscription`, {params: options}); return { ...resp.data, query: options.query || null } }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) const initialState: GetCustomerSubscriptionsState = { status: 'idle', bookmarks: null, totalPages: null, totalRecords: null, currentPage: null, limit: null, query: null, errorMessage: null } const getCustomerSubscriptionsSlice = createSlice({ name: 'getBookmarksByCollectionId', initialState, reducers: {}, extraReducers: (builder) => { builder.addCase(getCustomerSubscriptions.pending, (state) => { state.status = 'pending'; state.bookmarks = null; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.query = null; state.errorMessage = null; }); builder.addCase(getCustomerSubscriptions.fulfilled, (state, action) => { state.status = 'succeeded'; state.bookmarks = action.payload.bookmarks; state.totalRecords = action.payload.totalRecords; state.totalPages = action.payload.totalPages; state.limit = action.payload.limit; state.query = action.payload.query; state.currentPage = action.payload.currentPage; state.errorMessage = null; }); builder.addCase(getCustomerSubscriptions.rejected, (state, action) => { state.status = 'failed'; state.bookmarks = null; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.query = null; state.errorMessage = action.error.message || "Something went wrong"; }); } }) export const selectGetCustomerSubscriptions = (state: AppState) => state.getCustomerSubscriptions; export default getCustomerSubscriptionsSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/getFeaturedCollections.ts ================================================ import { createAsyncThunk, createSlice, PayloadAction } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { HYDRATE } from "next-redux-wrapper"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Collection } from "../types/Collection"; interface GetFeaturedCollectionsState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; collections: null | Collection[]; totalPages: number | null; totalRecords: number | null; currentPage: number | null; limit: number | null; errorMessage: null | string; } interface GetFeaturedCollectionsData { collections: Collection[]; totalPages: number; totalRecords: number; currentPage: number; limit: number } type GetFeaturedCollectionsResPayload = GetFeaturedCollectionsData; export const getFeaturedCollections = createAsyncThunk('/collection/public/featured', async () => { try{ const resp = await api.get('/collection/my-collections'); return resp.data; }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) const initialState: GetFeaturedCollectionsState = { status: 'idle', collections: null, totalPages: null, totalRecords: null, currentPage: null, limit: null, errorMessage: null } const getFeaturedCollectionsSlice = createSlice({ name: 'getFeaturedCollections', initialState, reducers: { setGetFeaturedCollections: (state, action) => { state.status = 'succeeded'; state.collections = action.payload.collections; state.totalPages = action.payload.totalPages; state.totalRecords = action.payload.totalRecords; state.currentPage = action.payload.currentPage; state.limit = action.payload.currentPage; state.errorMessage = null; } }, extraReducers: { [HYDRATE]: (state, action) => { if(action.payload.status !== 'idle') { state.status = 'succeeded'; state.collections = action.payload.getFeaturedCollections.collections; state.totalPages = action.payload.getFeaturedCollections.totalPages; state.totalRecords = action.payload.getFeaturedCollections.totalRecords; state.currentPage = action.payload.getFeaturedCollections.currentPage; state.limit = action.payload.getFeaturedCollections.limit; return state; } }, [getFeaturedCollections.pending as unknown as string]: (state) => { state.status = 'pending'; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.collections = null; state.errorMessage = null; return state; }, [getFeaturedCollections.fulfilled as unknown as string]: (state, action) => { state.status = 'succeeded'; state.collections = action.payload.collections; state.totalPages = action.payload.totalPages; state.totalRecords = action.payload.totalRecords; state.currentPage = action.payload.currentPage; state.limit = action.payload.limit; state.errorMessage = null; return state; }, [getFeaturedCollections.rejected as unknown as string]: (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || "Something went wrong"; state.totalPages = null; state.totalRecords = null; state.currentPage = null; state.limit = null; state.collections = null; return state; } } }) export const selectGetFeaturedCollections = (state: AppState) => state.getFeaturedCollections; export const { setGetFeaturedCollections } = getFeaturedCollectionsSlice.actions; export default getFeaturedCollectionsSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/getLoggedInCustCollections.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Collection } from "../types/Collection"; interface GetLoggedInCustCollectionsState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; collecitons: null | Collection[]; errorMessage: null | string; } export const getLoggedInCustCollections = createAsyncThunk('/collection/me', async () => { try{ const resp = await api.get('/collection/my-collections'); return resp.data; }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) const initialState: GetLoggedInCustCollectionsState = { status: 'idle', collecitons: null, errorMessage: null } const getLoggedInCustCollectionsSlice = createSlice({ name: 'getLoggedInCustCollections', initialState, reducers: {}, extraReducers: (builder) => { builder.addCase(getLoggedInCustCollections.pending, (state) => { state.status = 'pending'; state.collecitons = null; state.errorMessage = null; }); builder.addCase(getLoggedInCustCollections.fulfilled, (state, action) => { state.status = 'succeeded'; state.collecitons = action.payload; state.errorMessage = null; }); builder.addCase(getLoggedInCustCollections.rejected, (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || "Something went wrong"; state.collecitons = null; }); } }) export const selectGetLoggedInCustCollections = (state: AppState) => state.getLoggedInCustCollections; export default getLoggedInCustCollectionsSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/getSubscribedCollections.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Collection } from "../types/Collection"; interface GetSubscribedCollectionsState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; collecitons: null | Collection[]; errorMessage: null | string; } export const getSubscribedCollections = createAsyncThunk('/subscription/collections', async () => { try{ const resp = await api.get('/subscription/collections'); return resp.data; }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) const initialState: GetSubscribedCollectionsState = { status: 'idle', collecitons: null, errorMessage: null } const getSubscribedCollectionsSlice = createSlice({ name: 'getLoggedInCustCollections', initialState, reducers: {}, extraReducers: (builder) => { builder.addCase(getSubscribedCollections.pending, (state) => { state.status = 'pending'; state.collecitons = null; state.errorMessage = null; }); builder.addCase(getSubscribedCollections.fulfilled, (state, action) => { state.status = 'succeeded'; state.collecitons = action.payload; state.errorMessage = null; }); builder.addCase(getSubscribedCollections.rejected, (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || "Something went wrong"; state.collecitons = null; }); } }) export const selectGetSubscribedCollections = (state: AppState) => state.getSubscribedCollections; export default getSubscribedCollectionsSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/isCollectionSubscribed.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { AsyncThunkSubmissionError } from "../errors/AsyncThunkSubmissionError"; import { CollectionSubscription } from "../types/CollectionSubscription"; import { ValidationErrors } from "../types/validationErrors"; import { createSubmissionErrorFromErrObj } from "../utils/createError"; interface IsCollectionSubscribedResPayload { isSubscribed: false } interface IsCollectionSubscribedState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; isSubscribed: null | boolean; errorMessage: null |string; } const initialState: IsCollectionSubscribedState = { status: 'idle', isSubscribed: null, errorMessage: null, } export const isCollectionSubscribed = createAsyncThunk('/subscribe/collection-id/is-subscribed', async (collectionId) => { try { const res = await api.post(`/subscription/${collectionId}/is-subscribed`); return res.data.isSubscribed; } catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }); export const isCollectionSubscribedSlice = createSlice({ name: 'subscribeCollectionById', initialState, reducers: {}, extraReducers: async (builder) => { builder.addCase(isCollectionSubscribed.pending, (state, action) => { state.isSubscribed = null; state.errorMessage = null; state.status = "pending"; }) builder.addCase(isCollectionSubscribed.fulfilled, (state, action) => { state.isSubscribed = action.payload; state.status = "succeeded"; state.errorMessage = null; }) builder.addCase(isCollectionSubscribed.rejected, (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || null; state.isSubscribed = null; }) } }) export const selectIsCollectionSubscribed = (state: AppState) => state.isCollectionSubscribed; export default isCollectionSubscribedSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/subscribeCollectionById.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AppState } from "../.."; import api from "../../../utils/api"; import { AsyncThunkSubmissionError } from "../errors/AsyncThunkSubmissionError"; import { CollectionSubscription } from "../types/CollectionSubscription"; import { ValidationErrors } from "../types/validationErrors"; import { createSubmissionErrorFromErrObj } from "../utils/createError"; interface SubscribeCollectionByIdReqPayload { collectionId: string; }; type SubscribeCollectionByIdResPayload = CollectionSubscription; type SubscribeCollectionByIdValidationErrors = ValidationErrors; interface SubscribeCollectionByIdState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; collectionSubscription: null | CollectionSubscription; errorMessage: null |string; errors: SubscribeCollectionByIdValidationErrors | null; } const initialState: SubscribeCollectionByIdState = { status: 'idle', collectionSubscription: null, errorMessage: null, errors: null } export const subscribeCollectionById = createAsyncThunk}>('/subscribe/collection-id/add', async (collectionId, thunkApi) => { try { const res = await api.post(`/subscription/${collectionId}/add`); return res.data; } catch(e) { let err: AsyncThunkSubmissionError= new AsyncThunkSubmissionError("Something went wrong", null); if(e instanceof Error) { err = createSubmissionErrorFromErrObj(e); } return thunkApi.rejectWithValue(err); } }); export const subscribeCollectionByIdSlice = createSlice({ name: 'subscribeCollectionById', initialState, reducers: {}, extraReducers: async (builder) => { builder.addCase(subscribeCollectionById.pending, (state, action) => { state.collectionSubscription = null; state.errorMessage = null; state.errors = null; state.status = "pending"; }) builder.addCase(subscribeCollectionById.fulfilled, (state, action) => { state.collectionSubscription = action.payload; state.status = "succeeded"; state.errorMessage = null; state.errors = null; }) builder.addCase(subscribeCollectionById.rejected, (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || null; state.collectionSubscription = null; if(action.payload) { state.errorMessage = action.payload.message; state.errors = action.payload.validationErrors; } }) } }) export const selectSubscribeCollectionById = (state: AppState) => state.subscribeCollectionById; export default subscribeCollectionByIdSlice.reducer; ================================================ FILE: aquila/view/store/slices/collection/unSubscribeCollectionById.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { CollectionSubscription } from "../types/CollectionSubscription"; type UnSubscribeCollectionByIdResPayload = CollectionSubscription; interface UnSubscribeCollectionByIdState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; collectionSubscription: null | CollectionSubscription; errorMessage: null |string; } const initialState: UnSubscribeCollectionByIdState = { status: 'idle', collectionSubscription: null, errorMessage: null, } export const unSubscribeCollectionById = createAsyncThunk('/subscribe/collection-id/remove', async (collectionId, thunkApi) => { try { const res = await api.post(`/subscription/${collectionId}/remove`); return res.data; } catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }); export const unSubscribeCollectionByIdSlice = createSlice({ name: 'unSubscribeCollectionById', initialState, reducers: {}, extraReducers: async (builder) => { builder.addCase(unSubscribeCollectionById.pending, (state, action) => { state.collectionSubscription = null; state.errorMessage = null; state.status = "pending"; }) builder.addCase(unSubscribeCollectionById.fulfilled, (state, action) => { state.collectionSubscription = action.payload; state.status = "succeeded"; state.errorMessage = null; }) builder.addCase(unSubscribeCollectionById.rejected, (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || null; state.collectionSubscription = null; }) } }) export const selectUnSubscribeCollectionById = (state: AppState) => state.unSubscribeCollectionById; export default unSubscribeCollectionByIdSlice.reducer; ================================================ FILE: aquila/view/store/slices/customer/activateCustomer.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AppState } from "../.."; import api from "../../../utils/api"; import { AsyncThunkSubmissionError } from "../errors/AsyncThunkSubmissionError"; import { Customer } from "../types/Customer"; import { ValidationErrors } from "../types/validationErrors"; import { createSubmissionErrorFromErrObj } from "../utils/createError"; export interface ActivateCustomerData { firstName: string; lastName: string; email: string; desc: string; lightningAddress: string; } type ActivateCustomerReqPayload = ActivateCustomerData; type ActivateCustomerResPayload = Customer; type ActivateCustomerValidationErrors = ValidationErrors; interface ActivateCustomerState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; customer: null | Customer; errorMessage: null |string; errors: ActivateCustomerValidationErrors | null; } const initialState: ActivateCustomerState = { status: 'idle', customer: null, errorMessage: null, errors: null } export const activateCustomer = createAsyncThunk}>('/customer/activate', async (data, thunkApi) => { try { const res = await api.post('/customer/activate', data); return res.data; } catch(e) { let err: AsyncThunkSubmissionError= new AsyncThunkSubmissionError("Something went wrong", null); if(e instanceof Error) { err = createSubmissionErrorFromErrObj(e); } return thunkApi.rejectWithValue(err); } }); export const activateCustomerSlice = createSlice({ name: 'activateCustomer', initialState, reducers: {}, extraReducers: async (builder) => { builder.addCase(activateCustomer.pending, (state, action) => { state.customer = null; state.errorMessage = null; state.errors = null; state.status = "pending"; }) builder.addCase(activateCustomer.fulfilled, (state, action) => { state.customer = action.payload; state.status = "succeeded"; state.errorMessage = null; state.errors = null; }) builder.addCase(activateCustomer.rejected, (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || null; state.customer = null; if(action.payload) { state.errorMessage = action.payload.message; state.errors = action.payload.validationErrors; } }) } }) export const selectActivateCustomer = (state: AppState) => state.activateCustomer; export default activateCustomerSlice.reducer; ================================================ FILE: aquila/view/store/slices/customer/getCurrentLoggedInCustomer.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Customer } from "../types/Customer"; interface GetCurrentLoggedInCustomerState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; customer: null | Customer; errorMessage: null |string; } const initialState: GetCurrentLoggedInCustomerState = { status: 'idle', customer: null, errorMessage: null } export const getCurrentLoggedInCustomer = createAsyncThunk('/customer/getCurrentLoggedInCustomer', async () => { try{ const resp = await api.get('/customer/me'); return resp.data; }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) export const getCurrentLoggedInCustomerSlice = createSlice({ name: 'getCurrentLoggedInCustomer', initialState, reducers: {}, extraReducers: (builder) => { builder.addCase(getCurrentLoggedInCustomer.pending, (state) => { state.status = 'pending'; state.customer = null; state.errorMessage = null; }); builder.addCase(getCurrentLoggedInCustomer.fulfilled, (state, action) => { state.customer = action.payload; state.status = "succeeded"; state.errorMessage = null; }); builder.addCase(getCurrentLoggedInCustomer.rejected, (state, action) => { state.status = "failed", state.errorMessage = action.error.message || "Something went wrong", state.customer = null; }) } }); export const selectGetCurrentLoggedInCustomer = (state: AppState) => state.getCurrentLoggedInCustomer; export default getCurrentLoggedInCustomerSlice.reducer; ================================================ FILE: aquila/view/store/slices/customer/getCustomerById.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AxiosError } from "axios"; import { AppState } from "../.."; import api from "../../../utils/api"; import { Customer } from "../types/Customer"; interface GetCustomerByIdState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; customer: null | Customer; errorMessage: null |string; } const initialState: GetCustomerByIdState = { status: 'idle', customer: null, errorMessage: null } export const getCustomerById = createAsyncThunk('/customer/public/customerId', async (customerId: string) => { try{ const resp = await api.get(`/customer/public/${customerId}`); return resp.data; }catch(e) { let message = "Something went wrong"; if(e instanceof AxiosError) { message = e.response?.data.message || message; } throw new Error(message); } }) export const getCustomerByIdSlice = createSlice({ name: 'getCustomerById', initialState, reducers: {}, extraReducers: (builder) => { builder.addCase(getCustomerById.pending, (state) => { state.status = 'pending'; state.customer = null; state.errorMessage = null; }); builder.addCase(getCustomerById.fulfilled, (state, action) => { state.customer = action.payload; state.status = "succeeded"; state.errorMessage = null; }); builder.addCase(getCustomerById.rejected, (state, action) => { state.status = "failed", state.errorMessage = action.error.message || "Something went wrong", state.customer = null; }) } }); export const selectGetCustomerById = (state: AppState) => state.getCustomerById; export default getCustomerByIdSlice.reducer; ================================================ FILE: aquila/view/store/slices/customer/updateCustomer.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AppState } from "../.."; import api from "../../../utils/api"; import { AsyncThunkSubmissionError } from "../errors/AsyncThunkSubmissionError"; import { Customer } from "../types/Customer"; import { ValidationErrors } from "../types/validationErrors"; import { createSubmissionErrorFromErrObj } from "../utils/createError"; export interface UpdateCustomerData { firstName: string; lastName: string; email: string; desc: string; lightningAddress: string; } type UpdateCustomerReqPayload = UpdateCustomerData; type UpdateCustomerResPayload = Customer; type UpdateCustomerValidationErrors = ValidationErrors; interface UpdateCustomerState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; customer: null | Customer; errorMessage: null |string; errors: UpdateCustomerValidationErrors | null; } const initialState: UpdateCustomerState = { status: 'idle', customer: null, errorMessage: null, errors: null } export const updateCustomer = createAsyncThunk}>('/customer/activate', async (data, thunkApi) => { try { const res = await api.patch('/customer', data); return res.data; } catch(e) { let err: AsyncThunkSubmissionError= new AsyncThunkSubmissionError("Something went wrong", null); if(e instanceof Error) { err = createSubmissionErrorFromErrObj(e); } return thunkApi.rejectWithValue(err); } }); export const updateCustomerSlice = createSlice({ name: 'updateCustomer', initialState, reducers: {}, extraReducers: async (builder) => { builder.addCase(updateCustomer.pending, (state, action) => { state.customer = null; state.errorMessage = null; state.errors = null; state.status = "pending"; }) builder.addCase(updateCustomer.fulfilled, (state, action) => { state.customer = action.payload; state.status = "succeeded"; state.errorMessage = null; state.errors = null; }) builder.addCase(updateCustomer.rejected, (state, action) => { state.status = 'failed'; state.errorMessage = action.error.message || null; state.customer = null; if(action.payload) { state.errorMessage = action.payload.message; state.errors = action.payload.validationErrors; } }) } }) export const selectUpdateCustomer = (state: AppState) => state.updateCustomer; export default updateCustomerSlice.reducer; ================================================ FILE: aquila/view/store/slices/errors/AsyncThunkSubmissionError.ts ================================================ export class AsyncThunkSubmissionError extends Error { public validationErrors: T | null; constructor(message: string, validationErrors: T | null) { super(message) this.validationErrors = validationErrors; } } ================================================ FILE: aquila/view/store/slices/generateName.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AppState } from ".."; import api from "../../utils/api"; export const fetchNames = createAsyncThunk('customer/generateName', async () => { const response = await api.get('/customer/generate-name'); return response.data; }) interface GeneratedNameState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; firstName: string | null; lastName: string | null; } const initialState: GeneratedNameState = { status: 'idle', firstName: null, lastName: null } export const generateNameSlice = createSlice({ name: 'generatedName', initialState, reducers: { removeGeneratedNames: (state) => { state.status = 'idle'; state.firstName = null; state.lastName = null; } }, extraReducers: (builder) => { builder.addCase(fetchNames.fulfilled, (state, action) => { state.status = 'succeeded'; state.firstName = action.payload.firstName; state.lastName = action.payload.lastName; }) } }); export const { removeGeneratedNames } = generateNameSlice.actions; export const selectGeneratedName = (state: AppState) => state.generatedName; export default generateNameSlice.reducer; ================================================ FILE: aquila/view/store/slices/signup.ts ================================================ import { createAsyncThunk, createSlice } from "@reduxjs/toolkit"; import { AppState } from ".."; import api from "../../utils/api"; import { AsyncThunkSubmissionError } from "./errors/AsyncThunkSubmissionError"; import { Customer } from "./types/Customer"; import { ValidationErrors } from "./types/validationErrors"; import { createSubmissionErrorFromErrObj } from "./utils/createError"; interface Collection { id: string; name: string; desc: string; customerId: string; aquilaDbName: string; isSharable: true; indexedDocsCount: number; createdAt: string; updatedat: string; } interface SignUpData { firstName: string; lastName: string; } type SignUpRequestPayload = SignUpData; interface SignUpResponsePayload { customer: Customer; collection: Collection ; } type SignUpValidationErrors = ValidationErrors interface SignUpState { status: 'idle' | 'pending' | 'succeeded' | 'failed'; customer: Customer | null errors: SignUpValidationErrors | null; errorMessage: string | null; } const initialState: SignUpState = { status: 'idle', customer: null, errors: null, errorMessage: null } export const signUp = createAsyncThunk}>("/customer/sign-up", async (data, thunkApi) => { try { const res = await api.post('/customer', data); return res.data.customer; }catch(e) { let err: AsyncThunkSubmissionError= new AsyncThunkSubmissionError("Something went wrong", null); if(e instanceof Error) { err = createSubmissionErrorFromErrObj(e); } return thunkApi.rejectWithValue(err); } }) export const signUpSlice = createSlice({ name: 'signUp', initialState, reducers: { removeSignUpData: (state) => { state.status = 'idle'; state.customer = null; state.errors = null; state.errorMessage = null; } }, extraReducers: async (builder) => { builder.addCase(signUp.fulfilled, (state, action) => { if(action.payload) { state.customer = action.payload; state.errors = null; state.status = 'succeeded'; } }); builder.addCase(signUp.pending, (state) => { state.customer = null; state.errorMessage = null; state.status = "pending"; state.errors = null; }) builder.addCase(signUp.rejected, (state, action) => { state.status = 'failed'; state.customer = null; state.errorMessage = action.error.message || null; if(action.payload) { state.errorMessage = action.payload.message; state.errors = action.payload.validationErrors; } }); } }); export const selectSignUp = (state: AppState) => state.signUp; export const { removeSignUpData } = signUpSlice.actions; export default signUpSlice.reducer; ================================================ FILE: aquila/view/store/slices/types/Bookmark.ts ================================================ import { Settings } from "http2"; export enum BookmarkStatus { NOT_PROCESSED = 'NOT_PROCESSED', PROCESSING = 'PROCESSING', PROCESSED = 'PROCESSED' } export interface Bookmark { id: string; collectionId: string; url: string; html: string; title: string; author: string; coverImg: string; summary: string; description?: string; links: string; isHidden: boolean; status: BookmarkStatus createdAt: string; } ================================================ FILE: aquila/view/store/slices/types/Collection.ts ================================================ import { Customer } from "./Customer"; export interface Collection { id: string; name: string; desc: string; customerId: string; customer?: Customer; } ================================================ FILE: aquila/view/store/slices/types/CollectionSubscription.ts ================================================ export interface CollectionSubscription { id: string; collectionId: string; subscriberId: string; subscribedAt: string; createdAt: string; updatedAt: string; } ================================================ FILE: aquila/view/store/slices/types/Customer.ts ================================================ export interface Customer { id: string; firstName: string; lastName: string; email: string; avatar: string; secretKey: string; desc: string; lightningAddress: string; isActive: boolean; createdAt: string; updatedAt: string; } ================================================ FILE: aquila/view/store/slices/types/validationErrors.ts ================================================ interface ValidationErrorData { value: string; msg: string; param: string; location: string; } export type ResponsePayloadValidationErrors = Partial> export type ValidationErrors = Partial> ================================================ FILE: aquila/view/store/slices/utils/createError.ts ================================================ import { AxiosError } from "axios"; import { AsyncThunkSubmissionError } from "../errors/AsyncThunkSubmissionError"; const isArrayFieldError = (item: any) => { const match = item.param.match(/\[(\d+)\]/); if (match && match.length === 2) { return true; } return false; }; const isObjectType = (item: any) => { const split = item.param.split('.'); if (split.length === 2) { return true; } return false; }; const parseArrayFieldError = (item: any) => { const pattern = /\[(\d+)\]/; const match = item.param.match(pattern); const index = parseInt(match[1], 10); const param = item.param.replace(pattern, ''); const [key, prop] = param.split('.'); return { index, key, prop }; }; const collectValidationErrors = (errors: any) => { const formErrors: any = {}; errors.forEach((item: any) => { if (isArrayFieldError(item)) { const { key, index, prop } = parseArrayFieldError(item); if (!formErrors[key]) { formErrors[key] = []; formErrors[key][index] = {}; formErrors[key][index][prop] = item.msg; } else { if (!formErrors[key][index]) { formErrors[key][index] = {}; } formErrors[key][index][prop] = item.msg; } } else if (isObjectType(item)) { const [key, prop] = item.param.split('.'); if (!formErrors[key]) { formErrors[key] = {}; formErrors[key][prop] = item.msg; } else { if (!formErrors[key][prop]) { formErrors[key][prop] = item.msg; } } } else if (!formErrors[item]) { formErrors[item.param] = item.msg; } }); return formErrors; }; interface SubmissionErrorResponse { message: string; errors: Array; } export const createSubmissionErrorFromErrObj = (error: AxiosError> | Error) => { let message = "Something went wrong"; let validationErrors: T | null = null; if(error instanceof AxiosError) { message = error.response?.data.message || message; if(error.response?.status === 400 && error.response.data.errors) { validationErrors = collectValidationErrors(error.response.data.errors); } } const errObj = new AsyncThunkSubmissionError(message, validationErrors); return errObj; } ================================================ FILE: aquila/view/styles/globals.scss ================================================ // @import url('https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap'); @import url('https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;1,100;1,200;1,300;1,400;1,500;1,600;1,700&display=swap'); /* http://meyerweb.com/eric/tools/css/reset/ v2.0 | 20110126 License: none (public domain) */ html, body, div, span, applet, object, iframe, h1, h2, h3, h4, h5, h6, p, blockquote, pre, a, abbr, acronym, address, big, cite, code, del, dfn, em, img, ins, kbd, q, s, samp, small, strike, strong, sub, sup, tt, var, b, u, i, center, dl, dt, dd, ol, ul, li, fieldset, form, label, legend, table, caption, tbody, tfoot, thead, tr, th, td, article, aside, canvas, details, embed, figure, figcaption, footer, header, hgroup, menu, nav, output, ruby, section, summary, time, mark, audio, video { margin: 0; padding: 0; border: 0; font-size: 100%; vertical-align: baseline; } /* HTML5 display-role reset for older browsers */ article, aside, details, figcaption, figure, footer, header, hgroup, menu, nav, section { display: block; } body { line-height: 1; } ol, ul { list-style: none; } blockquote, q { quotes: none; } blockquote:before, blockquote:after, q:before, q:after { content: ''; content: none; } table { border-collapse: collapse; border-spacing: 0; } * { // font-family: 'Roboto', sans-serif; font-family: 'IBM Plex Sans', sans-serif; } html, body { height: 100%; min-height: 100%; } #__next { height: 100%; min-height: 100%; } ================================================ FILE: aquila/view/tsconfig.json ================================================ { "compilerOptions": { "target": "es5", "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, "strict": true, "forceConsistentCasingInFileNames": true, "noEmit": true, "esModuleInterop": true, "module": "esnext", "moduleResolution": "node", "resolveJsonModule": true, "isolatedModules": true, "jsx": "preserve", "incremental": true }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"] } ================================================ FILE: aquila/view/utils/api.ts ================================================ import axios from "axios"; import { getSession, signOut } from "next-auth/react"; import { store } from "../store"; const api = axios.create({ baseURL: process.env.NEXT_PUBLIC_AQUILA_API_URL, }); api.interceptors.request.use(async (config) => { const authState = store.getState().auth; let accessToken = authState.token; // console.log(authState); if(!accessToken && authState.status === "idle" && typeof window !== "undefined") { const session = await getSession(); // console.log(session, "Output"); if(session) { accessToken = session?.user.token || null; } } if(config.headers && accessToken) { config.headers.Authorization = `Bearer ${accessToken}`; } return config; }) api.interceptors.response.use((request) => request, (err) => { if(!err.response || err.response.data.code === 401) { signOut(); } }) export default api;