Repository: tardis-dev/tardis-node
Branch: master
Commit: cd69e6ecab40
Files: 141
Total size: 8.7 MB
Directory structure:
gitextract_eif38n1w/
├── .github/
│ └── workflows/
│ ├── ci.yaml
│ ├── npm_audit.yaml
│ └── publish.yaml
├── .gitignore
├── .npmrc
├── .prettierignore
├── .prettierrc
├── ADD_NEW_EXCHANGE.md
├── AGENTS.md
├── ARCHITECTURE.md
├── CLAUDE.md
├── LICENSE
├── README.md
├── example.js
├── package.json
├── src/
│ ├── apikeyaccessinfo.ts
│ ├── binarysplit.ts
│ ├── clearcache.ts
│ ├── combine.ts
│ ├── computable/
│ │ ├── booksnapshot.ts
│ │ ├── computable.ts
│ │ ├── index.ts
│ │ └── tradebar.ts
│ ├── consts.ts
│ ├── debug.ts
│ ├── downloaddatasets.ts
│ ├── exchangedetails.ts
│ ├── filter.ts
│ ├── handy.ts
│ ├── index.ts
│ ├── instrumentinfo.ts
│ ├── mappers/
│ │ ├── ascendex.ts
│ │ ├── binance.ts
│ │ ├── binancedex.ts
│ │ ├── binanceeuropeanoptions.ts
│ │ ├── bitfinex.ts
│ │ ├── bitflyer.ts
│ │ ├── bitget.ts
│ │ ├── bitmex.ts
│ │ ├── bitnomial.ts
│ │ ├── bitstamp.ts
│ │ ├── blockchaincom.ts
│ │ ├── bybit.ts
│ │ ├── bybitspot.ts
│ │ ├── coinbase.ts
│ │ ├── coinbaseinternational.ts
│ │ ├── coinflex.ts
│ │ ├── cryptocom.ts
│ │ ├── cryptofacilities.ts
│ │ ├── delta.ts
│ │ ├── deribit.ts
│ │ ├── dydx.ts
│ │ ├── dydxv4.ts
│ │ ├── ftx.ts
│ │ ├── gateio.ts
│ │ ├── gateiofutures.ts
│ │ ├── gemini.ts
│ │ ├── hitbtc.ts
│ │ ├── huobi.ts
│ │ ├── hyperliquid.ts
│ │ ├── index.ts
│ │ ├── kraken.ts
│ │ ├── kucoin.ts
│ │ ├── kucoinfutures.ts
│ │ ├── mapper.ts
│ │ ├── okex.ts
│ │ ├── okexspreads.ts
│ │ ├── phemex.ts
│ │ ├── poloniex.ts
│ │ ├── serum.ts
│ │ ├── upbit.ts
│ │ └── woox.ts
│ ├── options.ts
│ ├── orderbook.ts
│ ├── realtimefeeds/
│ │ ├── ascendex.ts
│ │ ├── binance.ts
│ │ ├── binancedex.ts
│ │ ├── binanceeuropeanoptions.ts
│ │ ├── bitfinex.ts
│ │ ├── bitflyer.ts
│ │ ├── bitget.ts
│ │ ├── bitmex.ts
│ │ ├── bitnomial.ts
│ │ ├── bitstamp.ts
│ │ ├── blockchaincom.ts
│ │ ├── bybit.ts
│ │ ├── coinbase.ts
│ │ ├── coinbaseinternational.ts
│ │ ├── coinflex.ts
│ │ ├── cryptocom.ts
│ │ ├── cryptofacilities.ts
│ │ ├── delta.ts
│ │ ├── deribit.ts
│ │ ├── dydx.ts
│ │ ├── dydx_v4.ts
│ │ ├── ftx.ts
│ │ ├── gateio.ts
│ │ ├── gateiofutures.ts
│ │ ├── gemini.ts
│ │ ├── hitbtc.ts
│ │ ├── huobi.ts
│ │ ├── hyperliquid.ts
│ │ ├── index.ts
│ │ ├── kraken.ts
│ │ ├── kucoin.ts
│ │ ├── kucoinfutures.ts
│ │ ├── mango.ts
│ │ ├── okex.ts
│ │ ├── okexspreads.ts
│ │ ├── phemex.ts
│ │ ├── poloniex.ts
│ │ ├── realtimefeed.ts
│ │ ├── serum.ts
│ │ ├── staratlas.ts
│ │ ├── upbit.ts
│ │ └── woox.ts
│ ├── replay.ts
│ ├── stream.ts
│ ├── types.ts
│ └── worker.ts
├── test/
│ ├── __snapshots__/
│ │ ├── combine.test.ts.snap
│ │ ├── compute.test.ts.snap
│ │ ├── mappers.test.ts.snap
│ │ └── replay.test.ts.snap
│ ├── binance-futures-split.live.test.ts
│ ├── binance-openinterest.live.test.ts
│ ├── binarysplit.test.ts
│ ├── combine.test.ts
│ ├── compute.test.ts
│ ├── downloaddatasets.test.ts
│ ├── gate-io-futures-decimal.live.test.ts
│ ├── httpclient.test.ts
│ ├── live.ts
│ ├── mappers.test.ts
│ ├── orderbook.test.ts
│ ├── package-exports.test.ts
│ ├── replay.test.ts
│ ├── setup.js
│ ├── stream.test.ts
│ └── tsconfig.json
└── tsconfig.json
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/workflows/ci.yaml
================================================
name: CI
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
ci:
name: CI
runs-on: ubuntu-latest
strategy:
matrix:
node-version: ['25.8.2']
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Use Node.js v${{ matrix.node-version }}
uses: actions/setup-node@v6
with:
node-version: ${{ matrix.node-version }}
- name: Install Dependencies And Compile TS
run: npm ci --ignore-scripts
- name: Verify Registry Signatures
run: npm audit signatures
- name: Audit Production Dependencies
run: npm audit --omit=dev --audit-level=critical
- name: Check Code Format
run: npm run check-format
- name: Run Tests
run: npm run test
================================================
FILE: .github/workflows/npm_audit.yaml
================================================
name: Full NPM Audit
on:
schedule:
- cron: '25 3 * * *'
workflow_dispatch:
permissions:
contents: read
jobs:
audit:
name: Full NPM Audit Report
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Use Node.js v25.8.2
uses: actions/setup-node@v6
with:
node-version: 25.8.2
- name: Generate Full Audit Report
id: audit
run: |
set +e
npm audit --package-lock-only --json > npm-audit.json
exit_code=$?
set -e
if [ ! -f npm-audit.json ]; then
echo '{}' > npm-audit.json
fi
echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
- name: Upload Audit Report
uses: actions/upload-artifact@v4
with:
name: npm-audit-report
path: npm-audit.json
- name: Summarize Audit Report
env:
AUDIT_EXIT_CODE: ${{ steps.audit.outputs.exit_code }}
run: |
node --input-type=module <<'EOF'
import fs from 'node:fs';
const report = JSON.parse(fs.readFileSync('npm-audit.json', 'utf8'));
const vulnerabilities = report.metadata?.vulnerabilities ?? {};
const lines = [
`Full npm audit exit code: ${process.env.AUDIT_EXIT_CODE}`,
`info: ${vulnerabilities.info ?? 0}`,
`low: ${vulnerabilities.low ?? 0}`,
`moderate: ${vulnerabilities.moderate ?? 0}`,
`high: ${vulnerabilities.high ?? 0}`,
`critical: ${vulnerabilities.critical ?? 0}`,
`total: ${vulnerabilities.total ?? 0}`,
'',
'Download the npm-audit-report artifact for the full JSON report.'
];
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join('\n')}\n`);
EOF
================================================
FILE: .github/workflows/publish.yaml
================================================
name: Publish New Release To NPM
on:
release:
# This specifies that the build will be triggered when we publish a release
types: [published]
permissions:
id-token: write
contents: write
jobs:
publish:
name: Publish New Release To NPM
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ github.event.release.target_commitish }}
- name: Use Node.js v25.8.2
uses: actions/setup-node@v6
with:
node-version: 25.8.2
registry-url: https://registry.npmjs.org/
- name: Install Dependencies And Compile TS
run: npm ci --ignore-scripts
- name: Verify Registry Signatures
run: npm audit signatures
- name: Audit Production Dependencies
run: npm audit --omit=dev --audit-level=critical
- name: Configure Git
run: |
git config --global user.name "GitHub Release Bot"
git config --global user.email "deploy@tardis.dev"
- name: Update package version
run: npm version ${{ github.event.release.tag_name }}
- name: Run Tests
run: npm run test
- name: Publish Package
run: npm publish
- name: Push Version Changes To GitHub
run: git push
================================================
FILE: .gitignore
================================================
node_modules
/dist
/*.log
.tardis-cache
*.tsbuildinfo
cache
bench/
.DS_Store
================================================
FILE: .npmrc
================================================
min-release-age=1
allow-git=none
================================================
FILE: .prettierignore
================================================
package.json
package-lock.json
yarn.lock
dist
================================================
FILE: .prettierrc
================================================
{
"printWidth": 140,
"semi": false,
"singleQuote": true,
"trailingComma": "none",
"endOfLine": "lf"
}
================================================
FILE: ADD_NEW_EXCHANGE.md
================================================
# Adding a New Exchange
## Overview
Adding an exchange to tardis-node requires three things: mappers (transform raw exchange messages into normalized types), a real-time feed (WebSocket connection), and constant definitions.
## Workflow
### 1. Add exchange constants
In `src/consts.ts`:
- Add exchange ID to the exchanges array
- Add channel info (list of available channels for the exchange)
### 2. Create mappers
Create `src/mappers/{exchange}.ts`. Each mapper class implements the Mapper interface — look at existing mapper implementations to find an exchange with a similar message format.
Mappers to implement depend on what the exchange provides: trades, book changes, tickers, derivative tickers, liquidations, book tickers, etc.
Register mapper factory in `src/mappers/index.ts`.
### 3. Create real-time feed
Create `src/realtimefeeds/{exchange}.ts`. Extend `RealTimeFeedBase` with:
- WebSocket URL
- Subscription message format
- Any exchange-specific hooks (decompression, heartbeat handling, error filtering)
Register in `src/realtimefeeds/index.ts`.
### 4. Test
Run tests and validation — see AGENTS.md for the full checklist.
## Decision Points
- **Date-based mapper versioning** — If the exchange changed its API format at some point, you may need different mapper implementations for different time periods. Look at existing examples in `src/mappers/index.ts` for the pattern.
- **Multi-connection feeds** — Some exchanges need multiple WebSocket connections. The base class supports this via `MultiConnectionRealTimeFeedBase`.
- **Decompression** — Some exchanges compress WebSocket messages. Override the decompress hook if needed.
- **Filter optimization** — The base class has `optimizeFilters()` for normalizing subscription filters. Override if the exchange needs special handling.
================================================
FILE: AGENTS.md
================================================
# tardis-node
Public npm package (`tardis-dev`). Provides async iterator API for historical replay and real-time streaming of cryptocurrency market data, with exchange-specific mappers for normalization.
## Build & Test
```bash
npm run build # tsc
npm test # build + jest
npm run check-format # prettier check
```
## Editing Rules
- Keep backward compatibility for public API signatures — this is a published npm package
- Maintain cache key stability (filters are normalized/sorted intentionally)
- Preserve memory-safe streaming behavior (avoid large in-memory buffering)
- Exchange additions must update realtime feed + mapper tables consistently
- **Format after every edit** — run `npx prettier --write` on modified files after each change
## Validation
- `npm run build && npm test`
- `npm run check-format`
## Operational Docs
- [ARCHITECTURE.md](ARCHITECTURE.md) — async iterators, replay pipeline, mapper system, order book
- [ADD_NEW_EXCHANGE.md](ADD_NEW_EXCHANGE.md) — add mappers and realtime feed for a new exchange
## Publishing
Published via GitHub Actions (`publish.yaml`). Do not publish manually unless explicitly requested.
## Keeping Docs Current
When you change code, check if any docs in this repo become stale as a result — if so, update them. When following a workflow doc, if the steps don't match reality, fix the doc so the next run is better.
================================================
FILE: ARCHITECTURE.md
================================================
# Architecture
tardis-node provides a unified async iterator API for consuming cryptocurrency market data. Two primary modes: **replay** (historical) and **stream** (real-time), both sharing the same normalized data types and mapper infrastructure.
## Core Design
Every data source produces an `AsyncIterableIterator`. This applies uniformly to raw replay, raw streaming, normalized replay, normalized streaming, combined streams, and computed/derived data.
## Replay Pipeline
```
Main Thread Worker Thread
│ │
│── Start replay ──→ │
│ Fetch data slice from API
│ Cache to disk (.gz file)
│ ←── message (sliceKey, path) ── │
│ Fetch next slice...
│ │
Read cached file from disk │
Decompress (gunzip) │
Split by newlines │
Parse JSON messages │
Yield {localTimestamp, message} │
```
Worker thread pre-fetches and caches slices while the main thread processes the current one. This keeps I/O and CPU pipelined.
## Real-time Streaming
`RealTimeFeedBase` manages WebSocket connections to exchanges. Handles connection lifecycle (connect, subscribe, validate, reconnect on failure). Exchange-specific feeds extend the base class with subscription formats and message handling.
## Mapper System
Mappers transform raw exchange messages into normalized types (trades, book changes, tickers, liquidations, etc.). Each exchange has mapper classes registered in `src/mappers/index.ts`.
Some exchanges have date-based mapper versioning — different mapper implementations for different time periods when the exchange changed its API format.
## Key Abstractions
- **`combine()`** — Merges multiple async iterables into one, ordered by timestamp. Enables cross-exchange data feeds.
- **`compute()`** — Wraps an async iterable and produces derived data (book snapshots, trade bars) via computables.
- **`OrderBook`** — Full limit order book reconstruction from incremental updates, using a Red-Black Tree for efficient price level management.
## Configuration
Exchange definitions and channel info in `src/consts.ts`. Mapper and feed registrations in their respective `index.ts` files.
================================================
FILE: CLAUDE.md
================================================
@AGENTS.md
================================================
FILE: LICENSE
================================================
Mozilla Public License Version 2.0
==================================
1. Definitions
--------------
1.1. "Contributor"
means each individual or legal entity that creates, contributes to
the creation of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used
by a Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached
the notice in Exhibit A, the Executable Form of such Source Code
Form, and Modifications of such Source Code Form, in each case
including portions thereof.
1.5. "Incompatible With Secondary Licenses"
means
(a) that the initial Contributor has attached the notice described
in Exhibit B to the Covered Software; or
(b) that the Covered Software was made available under the terms of
version 1.1 or earlier of the License, but not also under the
terms of a Secondary License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible,
whether at the time of the initial grant or subsequently, any and
all of the rights conveyed by this License.
1.10. "Modifications"
means any of the following:
(a) any file in Source Code Form that results from an addition to,
deletion from, or modification of the contents of Covered
Software; or
(b) any new file in Source Code Form that contains any Covered
Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method,
process, and apparatus claims, in any patent Licensable by such
Contributor that would be infringed, but for the grant of the
License, by the making, using, selling, offering for sale, having
made, import, or transfer of either its Contributions or its
Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU
Lesser General Public License, Version 2.1, the GNU Affero General
Public License, Version 3.0, or any later versions of those
licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this
License. For legal entities, "You" includes any entity that
controls, is controlled by, or is under common control with You. For
purposes of this definition, "control" means (a) the power, direct
or indirect, to cause the direction or management of such entity,
whether by contract or otherwise, or (b) ownership of more than
fifty percent (50%) of the outstanding shares or beneficial
ownership of such entity.
2. License Grants and Conditions
--------------------------------
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free,
non-exclusive license:
(a) under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available,
modify, display, perform, distribute, and otherwise exploit its
Contributions, either on an unmodified basis, with Modifications, or
as part of a Larger Work; and
(b) under Patent Claims of such Contributor to make, use, sell, offer
for sale, have made, import, and otherwise transfer either its
Contributions or its Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution
become effective for each Contribution on the date the Contributor first
distributes such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under
this License. No additional rights or licenses will be implied from the
distribution or licensing of Covered Software under this License.
Notwithstanding Section 2.1(b) above, no patent license is granted by a
Contributor:
(a) for any code that a Contributor has removed from Covered Software;
or
(b) for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
(c) under Patent Claims infringed by Covered Software in the absence of
its Contributions.
This License does not grant any rights in the trademarks, service marks,
or logos of any Contributor (except as may be necessary to comply with
the notice requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to
distribute the Covered Software under a subsequent version of this
License (see Section 10.2) or under the terms of a Secondary License (if
permitted under the terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its
Contributions are its original creation(s) or it has sufficient rights
to grant the rights to its Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under
applicable copyright doctrines of fair use, fair dealing, or other
equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
in Section 2.1.
3. Responsibilities
-------------------
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under
the terms of this License. You must inform recipients that the Source
Code Form of the Covered Software is governed by the terms of this
License, and how they can obtain a copy of this License. You may not
attempt to alter or restrict the recipients' rights in the Source Code
Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
(a) such Covered Software must also be made available in Source Code
Form, as described in Section 3.1, and You must inform recipients of
the Executable Form how they can obtain a copy of such Source Code
Form by reasonable means in a timely manner, at a charge no more
than the cost of distribution to the recipient; and
(b) You may distribute such Executable Form under the terms of this
License, or sublicense it under different terms, provided that the
license for the Executable Form does not attempt to limit or alter
the recipients' rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for
the Covered Software. If the Larger Work is a combination of Covered
Software with a work governed by one or more Secondary Licenses, and the
Covered Software is not Incompatible With Secondary Licenses, this
License permits You to additionally distribute such Covered Software
under the terms of such Secondary License(s), so that the recipient of
the Larger Work may, at their option, further distribute the Covered
Software under the terms of either this License or such Secondary
License(s).
3.4. Notices
You may not remove or alter the substance of any license notices
(including copyright notices, patent notices, disclaimers of warranty,
or limitations of liability) contained within the Source Code Form of
the Covered Software, except that You may alter any license notices to
the extent required to remedy known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support,
indemnity or liability obligations to one or more recipients of Covered
Software. However, You may do so only on Your own behalf, and not on
behalf of any Contributor. You must make it absolutely clear that any
such warranty, support, indemnity, or liability obligation is offered by
You alone, and You hereby agree to indemnify every Contributor for any
liability incurred by such Contributor as a result of warranty, support,
indemnity or liability terms You offer. You may include additional
disclaimers of warranty and limitations of liability specific to any
jurisdiction.
4. Inability to Comply Due to Statute or Regulation
---------------------------------------------------
If it is impossible for You to comply with any of the terms of this
License with respect to some or all of the Covered Software due to
statute, judicial order, or regulation then You must: (a) comply with
the terms of this License to the maximum extent possible; and (b)
describe the limitations and the code they affect. Such description must
be placed in a text file included with all distributions of the Covered
Software under this License. Except to the extent prohibited by statute
or regulation, such description must be sufficiently detailed for a
recipient of ordinary skill to be able to understand it.
5. Termination
--------------
5.1. The rights granted under this License will terminate automatically
if You fail to comply with any of its terms. However, if You become
compliant, then the rights granted under this License from a particular
Contributor are reinstated (a) provisionally, unless and until such
Contributor explicitly and finally terminates Your grants, and (b) on an
ongoing basis, if such Contributor fails to notify You of the
non-compliance by some reasonable means prior to 60 days after You have
come back into compliance. Moreover, Your grants from a particular
Contributor are reinstated on an ongoing basis if such Contributor
notifies You of the non-compliance by some reasonable means, this is the
first time You have received notice of non-compliance with this License
from such Contributor, and You become compliant prior to 30 days after
Your receipt of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions,
counter-claims, and cross-claims) alleging that a Contributor Version
directly or indirectly infringes any patent, then the rights granted to
You by any and all Contributors for the Covered Software under Section
2.1 of this License shall terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
end user license agreements (excluding distributors and resellers) which
have been validly granted by You or Your distributors under this License
prior to termination shall survive termination.
************************************************************************
* *
* 6. Disclaimer of Warranty *
* ------------------------- *
* *
* Covered Software is provided under this License on an "as is" *
* basis, without warranty of any kind, either expressed, implied, or *
* statutory, including, without limitation, warranties that the *
* Covered Software is free of defects, merchantable, fit for a *
* particular purpose or non-infringing. The entire risk as to the *
* quality and performance of the Covered Software is with You. *
* Should any Covered Software prove defective in any respect, You *
* (not any Contributor) assume the cost of any necessary servicing, *
* repair, or correction. This disclaimer of warranty constitutes an *
* essential part of this License. No use of any Covered Software is *
* authorized under this License except under this disclaimer. *
* *
************************************************************************
************************************************************************
* *
* 7. Limitation of Liability *
* -------------------------- *
* *
* Under no circumstances and under no legal theory, whether tort *
* (including negligence), contract, or otherwise, shall any *
* Contributor, or anyone who distributes Covered Software as *
* permitted above, be liable to You for any direct, indirect, *
* special, incidental, or consequential damages of any character *
* including, without limitation, damages for lost profits, loss of *
* goodwill, work stoppage, computer failure or malfunction, or any *
* and all other commercial damages or losses, even if such party *
* shall have been informed of the possibility of such damages. This *
* limitation of liability shall not apply to liability for death or *
* personal injury resulting from such party's negligence to the *
* extent applicable law prohibits such limitation. Some *
* jurisdictions do not allow the exclusion or limitation of *
* incidental or consequential damages, so this exclusion and *
* limitation may not apply to You. *
* *
************************************************************************
8. Litigation
-------------
Any litigation relating to this License may be brought only in the
courts of a jurisdiction where the defendant maintains its principal
place of business and such litigation shall be governed by laws of that
jurisdiction, without reference to its conflict-of-law provisions.
Nothing in this Section shall prevent a party's ability to bring
cross-claims or counter-claims.
9. Miscellaneous
----------------
This License represents the complete agreement concerning the subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. Any law or regulation which provides
that the language of a contract shall be construed against the drafter
shall not be used to construe this License against a Contributor.
10. Versions of the License
---------------------------
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section
10.3, no one other than the license steward has the right to modify or
publish new versions of this License. Each version will be given a
distinguishing version number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version
of the License under which You originally received the Covered Software,
or under the terms of any subsequent version published by the license
steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to
create a new license for such software, you may create and use a
modified version of this License if you rename the license and remove
any references to the name of the license steward (except to note that
such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses
If You choose to distribute Source Code Form that is Incompatible With
Secondary Licenses under the terms of this version of the License, the
notice described in Exhibit B of this License must be attached.
Exhibit A - Source Code Form License Notice
-------------------------------------------
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular
file, then You may include the notice in a location (such as a LICENSE
file in a relevant directory) where a recipient would be likely to look
for such a notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
This Source Code Form is "Incompatible With Secondary Licenses", as
defined by the Mozilla Public License, v. 2.0.
================================================
FILE: README.md
================================================
# tardis-dev
[](https://www.npmjs.org/package/tardis-dev)
Node.js `tardis-dev` library provides convenient access to tick-level real-time and historical cryptocurrency market data both in exchange native and normalized formats. Instead of callbacks it relies on [async iteration (for await ...of)](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) enabling composability features like [seamless switching between real-time data streaming and historical data replay](https://docs.tardis.dev/node-client/normalization#seamless-switching-between-real-time-streaming-and-historical-market-data-replay) or [computing derived data locally](https://docs.tardis.dev/node-client/normalization#computing-derived-data-locally).
```javascript
import { replayNormalized, normalizeTrades, normalizeBookChanges } from 'tardis-dev'
const messages = replayNormalized(
{
exchange: 'binance',
symbols: ['btcusdt'],
from: '2024-03-01',
to: '2024-03-02'
},
normalizeTrades,
normalizeBookChanges
)
for await (const message of messages) {
console.log(message)
}
```
## Features
- historical tick-level [market data replay](https://docs.tardis.dev/node-client/replaying-historical-data) backed by [tardis.dev HTTP API](https://docs.tardis.dev/api/http-api-reference#data-feeds-exchange) — includes full order book depth snapshots plus incremental updates, tick-by-tick trades, historical open interest, funding, index, mark prices, liquidations and more
- consolidated [real-time data streaming API](https://docs.tardis.dev/node-client/streaming-real-time-data) connecting directly to exchanges' public WebSocket APIs
- support for both [exchange-native and normalized market data](https://docs.tardis.dev/faq/data) formats (unified format for accessing market data across all supported exchanges — normalized trades, order book and ticker data)
- [seamless switching between real-time streaming and historical market data replay](https://docs.tardis.dev/node-client/normalization#seamless-switching-between-real-time-streaming-and-historical-market-data-replay) thanks to [`async iterables`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) providing unified way of consuming data messages
- transparent historical local data caching \(cached data is stored on disk per slice in compressed format and decompressed on demand when reading the data\)
- support for many cryptocurrency exchanges — see [docs.tardis.dev](https://docs.tardis.dev) for the full list
- automatic closed connections and stale connections reconnection logic for real-time streams
- [combining multiple exchanges feeds into single one](https://docs.tardis.dev/node-client/normalization#combining-data-streams) via [`combine`](https://docs.tardis.dev/node-client/normalization#combining-data-streams) helper function — synchronized historical market data replay and consolidated real-time data streaming from multiple exchanges
- [computing derived data locally](https://docs.tardis.dev/node-client/normalization#computing-derived-data-locally) like order book imbalance, custom trade bars, book snapshots and more via [`compute`](https://docs.tardis.dev/node-client/normalization#computing-derived-data-locally) helper function and `computables`, e.g., volume based bars, top 20 levels order book snapshots taken every 10 ms etc.
- [full limit order book reconstruction](https://docs.tardis.dev/node-client/normalization#limit-order-book-reconstruction) both for real-time and historical data via `OrderBook` object
- fast and lightweight architecture — low memory footprint and no heavy in-memory buffering
- [extensible mapping logic](https://docs.tardis.dev/node-client/normalization#modifying-built-in-and-adding-custom-normalizers) that allows adjusting normalized formats for specific needs
- [built-in TypeScript support](https://docs.tardis.dev/node-client/quickstart#es-modules-and-typescript)
## Installation
Requires Node.js v24+ installed.
```bash
npm install tardis-dev --save
```
`tardis-dev` is ESM-only. Examples in this README use ES modules and top-level await. Save snippets as `.mjs` or set `"type": "module"` in your `package.json`.
## Documentation
### [See official docs](https://docs.tardis.dev/node-client/quickstart).
## Examples
### Real-time spread across multiple exchanges
Example showing how to quickly display real-time spread and best bid/ask info across multiple exchanges at once. It can be easily adapted to do the same for historical data \(`replayNormalized` instead of `streamNormalized`).
```javascript
import { streamNormalized, normalizeBookChanges, combine, compute, computeBookSnapshots } from 'tardis-dev'
const exchangesToStream = [
{ exchange: 'bitmex', symbols: ['XBTUSD'] },
{ exchange: 'deribit', symbols: ['BTC-PERPETUAL'] },
{ exchange: 'cryptofacilities', symbols: ['PI_XBTUSD'] }
]
// for each specified exchange call streamNormalized for it
// so we have multiple real-time streams for all specified exchanges
const realTimeStreams = exchangesToStream.map((e) => {
return streamNormalized(e, normalizeBookChanges)
})
// combine all real-time message streams into one
const messages = combine(...realTimeStreams)
// create book snapshots with depth1 that are produced
// every time best bid/ask info is changed
// effectively computing real-time quotes
const realTimeQuoteComputable = computeBookSnapshots({
depth: 1,
interval: 0,
name: 'realtime_quote'
})
// compute real-time quotes for combines real-time messages
const messagesWithQuotes = compute(messages, realTimeQuoteComputable)
const spreads = {}
// print spreads info every 100ms
setInterval(() => {
console.clear()
console.log(spreads)
}, 100)
// update spreads info real-time
for await (const message of messagesWithQuotes) {
if (message.type === 'book_snapshot') {
spreads[message.exchange] = {
spread: message.asks[0].price - message.bids[0].price,
bestBid: message.bids[0],
bestAsk: message.asks[0]
}
}
}
```
### Seamless switching between real-time streaming and historical market data replay
Example showing simple pattern of providing `async iterable` of market data messages to the function that can process them no matter if it's is real-time or historical market data. That effectively enables having the same 'data pipeline' for backtesting and live trading.
```javascript
import { replayNormalized, streamNormalized, normalizeTrades, compute, computeTradeBars } from 'tardis-dev'
const historicalMessages = replayNormalized(
{
exchange: 'binance',
symbols: ['btcusdt'],
from: '2024-03-01',
to: '2024-03-02'
},
normalizeTrades
)
const realTimeMessages = streamNormalized(
{
exchange: 'binance',
symbols: ['btcusdt']
},
normalizeTrades
)
async function produceVolumeBasedTradeBars(messages) {
const withVolumeTradeBars = compute(
messages,
computeTradeBars({
kind: 'volume',
interval: 1 // aggregate by 1 BTC traded volume
})
)
for await (const message of withVolumeTradeBars) {
if (message.type === 'trade_bar') {
console.log(message.name, message)
}
}
}
await produceVolumeBasedTradeBars(historicalMessages)
// or for real time data
// await produceVolumeBasedTradeBars(realTimeMessages)
```
### Stream real-time market data in exchange native data format
```javascript
import { stream } from 'tardis-dev'
const messages = stream({
exchange: 'binance',
filters: [
{ channel: 'trade', symbols: ['btcusdt'] },
{ channel: 'depth', symbols: ['btcusdt'] }
]
})
for await (const { localTimestamp, message } of messages) {
console.log(localTimestamp, message)
}
```
### Replay historical market data in exchange native data format
```javascript
import { replay } from 'tardis-dev'
const messages = replay({
exchange: 'binance',
filters: [
{ channel: 'trade', symbols: ['btcusdt'] },
{ channel: 'depth', symbols: ['btcusdt'] }
],
from: '2024-03-01',
to: '2024-03-02'
})
for await (const { localTimestamp, message } of messages) {
console.log(localTimestamp, message)
}
```
## See the [tardis-dev docs](https://docs.tardis.dev/node-client/quickstart) for more examples.
================================================
FILE: example.js
================================================
import { replayNormalized, streamNormalized, normalizeTrades, compute, computeTradeBars } from 'tardis-dev'
const historicalMessages = replayNormalized(
{
exchange: 'binance',
symbols: ['btcusdt'],
from: '2024-03-01',
to: '2024-03-02'
},
normalizeTrades
)
const realTimeMessages = streamNormalized(
{
exchange: 'binance',
symbols: ['btcusdt']
},
normalizeTrades
)
async function produceVolumeBasedTradeBars(messages) {
// aggregate by 1 BTC traded volume
const withVolumeTradeBars = compute(messages, computeTradeBars({ kind: 'volume', interval: 1 }))
for await (const message of withVolumeTradeBars) {
if (message.type === 'trade_bar') {
console.log(message.name, message)
}
}
}
await produceVolumeBasedTradeBars(historicalMessages)
// or for real time data
// await produceVolumeBasedTradeBars(realTimeMessages)
================================================
FILE: package.json
================================================
{
"name": "tardis-dev",
"version": "16.0.0",
"engines": {
"node": ">=25"
},
"devEngines": {
"runtime": {
"name": "node",
"version": ">=25"
},
"packageManager": {
"name": "npm",
"version": ">=11.11.1"
}
},
"description": "Convenient access to tick-level historical and real-time cryptocurrency market data via Node.js",
"main": "dist/index.js",
"source": "src/index.ts",
"types": "dist/index.d.ts",
"type": "module",
"exports": {
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js",
"default": "./dist/index.js"
},
"./package.json": "./package.json"
},
"repository": "tardis-dev/tardis-node",
"homepage": "https://docs.tardis.dev/api/node-js",
"scripts": {
"build": "tsc -p tsconfig.json",
"precommit": "lint-staged",
"test": "npm run build && node --experimental-vm-modules ./node_modules/jest/bin/jest.js --forceExit --runInBand",
"prepare": "npm run build",
"format": "prettier --write .",
"check-format": "prettier --check ."
},
"files": [
"src",
"dist",
"example.js"
],
"keywords": [
"cryptocurrency data feed",
"market data",
"api client",
"crypto markets data replay",
"historical data",
"real-time cryptocurrency market data feed",
"historical cryptocurrency prices",
"cryptocurrency api",
"real-time normalized WebSocket cryptocurrency markets data",
"normalized cryptocurrency market data API",
"order book reconstruction",
"market data normalization",
"cryptocurrency api",
"cryptocurrency",
"orderbook",
"exchange",
"websocket",
"realtime",
"bitmex",
"binance",
"trading",
"high granularity order book data"
],
"license": "MPL-2.0",
"dependencies": {
"bintrees": "^1.0.2",
"debug": "^4.3.3",
"follow-redirects": "^1.15.9",
"https-proxy-agent": "^8.0.0",
"p-map": "^7.0.4",
"socks-proxy-agent": "^9.0.0",
"ws": "^8.18.3"
},
"devDependencies": {
"@types/bintrees": "^1.0.6",
"@types/debug": "^4.1.7",
"@types/follow-redirects": "^1.14.4",
"@types/jest": "^29.0.0",
"@types/node": "^25.3.5",
"@types/ws": "^8.18.1",
"jest": "^29.0.0",
"lint-staged": "^12.1.3",
"prettier": "^2.5.1",
"ts-jest": "^29.4.0",
"typescript": "^5.9.3"
},
"lint-staged": {
"*.{ts}": [
"prettier --write",
"git add"
]
},
"jest": {
"extensionsToTreatAsEsm": [
".ts"
],
"moduleNameMapper": {
"^(\\.{1,2}/.*)\\.js$": "$1"
},
"transform": {
"\\.(ts|tsx)?$": [
"ts-jest",
{
"useESM": true,
"tsconfig": "./test/tsconfig.json"
}
]
},
"testEnvironment": "node",
"setupFiles": [
"./test/setup.js"
]
},
"runkitExampleFilename": "example.js"
}
================================================
FILE: src/apikeyaccessinfo.ts
================================================
import { getJSON } from './handy.ts'
import { getOptions } from './options.ts'
import { Exchange } from './types.ts'
export async function getApiKeyAccessInfo(apiKey?: string) {
const options = getOptions()
const apiKeyToCheck = apiKey || options.apiKey
const { data } = await getJSON(`${options.endpoint}/api-key-info`, {
headers: {
Authorization: `Bearer ${apiKeyToCheck}`
}
})
return data
}
export type ApiKeyAccessInfo = {
exchange: Exchange
accessType: string
from: string
to: string
symbols: string[]
dataPlan: string
}[]
================================================
FILE: src/binarysplit.ts
================================================
import { Transform } from 'stream'
import type { TransformCallback } from 'stream'
// Inspired by https://github.com/maxogden/binary-split/blob/master/index.js
export class BinarySplitStream extends Transform {
private readonly _NEW_LINE_BYTE: number
private _buffered?: Buffer
constructor() {
super({
readableObjectMode: true
})
this._NEW_LINE_BYTE = 10
this._buffered = undefined
}
_transform(chunk: Buffer, _: string, callback: TransformCallback) {
let chunkStart = 0
if (this._buffered !== undefined) {
const firstNewLineIndex = chunk.indexOf(this._NEW_LINE_BYTE)
if (firstNewLineIndex === -1) {
this._buffered = Buffer.concat([this._buffered, chunk])
callback()
return
}
this.push(Buffer.concat([this._buffered, chunk.subarray(0, firstNewLineIndex)]))
this._buffered = undefined
chunkStart = firstNewLineIndex + 1
}
let offset = chunkStart
let lineStart = chunkStart
while (true) {
const newLineIndex = chunk.indexOf(this._NEW_LINE_BYTE, offset)
if (newLineIndex === -1) {
break
}
this.push(chunk.subarray(lineStart, newLineIndex))
offset = newLineIndex + 1
lineStart = offset
}
this._buffered = lineStart < chunk.length ? chunk.subarray(lineStart) : undefined
callback()
}
}
================================================
FILE: src/clearcache.ts
================================================
import { rmSync } from 'node:fs'
import { rm } from 'node:fs/promises'
import { debug } from './debug.ts'
import { getOptions } from './options.ts'
import { Filter, Exchange } from './types.ts'
import { sha256, optimizeFilters, doubleDigit } from './handy.ts'
export async function clearCache(exchange?: Exchange, filters?: Filter[], year?: number, month?: number, day?: number) {
try {
const dirToRemove = getDirToRemove(exchange, filters, year, month, day)
debug('clearing cache dir: %s', dirToRemove)
await rm(dirToRemove, { force: true, recursive: true })
debug('cleared cache dir: %s', dirToRemove)
} catch (e) {
debug('clearing cache dir error: %o', e)
}
}
export function clearCacheSync(exchange?: Exchange, filters?: Filter[], year?: number, month?: number, day?: number) {
try {
const dirToRemove = getDirToRemove(exchange, filters, year, month, day)
debug('clearing cache (sync) dir: %s', dirToRemove)
rmSync(dirToRemove, { force: true, recursive: true })
debug('cleared cache(sync) dir: %s', dirToRemove)
} catch (e) {
debug('clearing cache (sync) dir error: %o', e)
}
}
function getDirToRemove(exchange?: Exchange, filters?: Filter[], year?: number, month?: number, day?: number) {
const options = getOptions()
let dirToRemove = `${options.cacheDir}/feeds`
if (exchange !== undefined) {
dirToRemove += `/${exchange}`
}
if (filters !== undefined) {
dirToRemove += `/${sha256(optimizeFilters(filters))}`
}
if (year !== undefined) {
dirToRemove += `/${year}`
}
if (month !== undefined) {
dirToRemove += `/${doubleDigit(month)}`
}
if (day !== undefined) {
dirToRemove += `/${doubleDigit(day)}`
}
return dirToRemove
}
================================================
FILE: src/combine.ts
================================================
import { PassThrough } from 'stream'
import { once } from 'events'
type NextMessageResultWithIndex = {
index: number
result: IteratorResult
}
type Combinable = { localTimestamp: Date }
const DATE_MAX = new Date(8640000000000000)
type OffsetMS = number | ((message: Combinable) => number)
async function nextWithIndex(
iterator: AsyncIterableIterator | { stream: AsyncIterableIterator; offsetMS: OffsetMS },
index: number
): Promise {
if ('offsetMS' in iterator) {
const result = await iterator.stream.next()
if (!result.done) {
const offsetMS = typeof iterator.offsetMS === 'function' ? iterator.offsetMS(result.value) : iterator.offsetMS
if (offsetMS !== 0) {
result.value.localTimestamp.setUTCMilliseconds(result.value.localTimestamp.getUTCMilliseconds() + offsetMS)
}
}
return {
result,
index
}
} else {
const result = await iterator.next()
return {
result,
index
}
}
}
function findOldestResult(oldest: NextMessageResultWithIndex, current: NextMessageResultWithIndex) {
if (oldest.result.done) {
return oldest
}
if (current.result.done) {
return current
}
const currentTimestamp = current.result.value.localTimestamp.valueOf()
const oldestTimestamp = oldest.result.value.localTimestamp.valueOf()
if (currentTimestamp < oldestTimestamp) {
return current
}
if (currentTimestamp === oldestTimestamp) {
const currentTimestampMicroSeconds = current.result.value.localTimestamp.μs || 0
const oldestTimestampMicroSeconds = oldest.result.value.localTimestamp.μs || 0
if (currentTimestampMicroSeconds < oldestTimestampMicroSeconds) {
return current
}
}
return oldest
}
// combines multiple iterators from for example multiple exchanges
// works both for real-time and historical data
export async function* combine<
T extends AsyncIterableIterator[] | { stream: AsyncIterableIterator; offsetMS: OffsetMS }[]
>(
...iteratorsPayload: T
): AsyncIterableIterator<
T extends AsyncIterableIterator[] ? U : T extends { stream: AsyncIterableIterator }[] ? Z : never
> {
const iterators = iteratorsPayload.map((payload) => {
if ('stream' in payload) {
return payload.stream
}
return payload
})
if (iterators.length === 0) {
return
}
// decide based on first provided iterator if we're dealing with real-time or historical data streams
if ((iterators[0] as any).__realtime__) {
const combinedStream = new PassThrough({
objectMode: true,
highWaterMark: 8096
})
combinedStream.setMaxListeners(iterators.length + 1)
iterators.forEach(async function writeMessagesToCombinedStream(messages) {
for await (const message of messages) {
if (combinedStream.destroyed) {
return
}
if (!combinedStream.write(message)) {
// Handle backpressure on write
await once(combinedStream, 'drain')
}
}
})
for await (const message of combinedStream) {
yield message
}
} else {
return yield* combineHistorical(iteratorsPayload) as any
}
}
async function* combineHistorical(
iterators: AsyncIterableIterator[] | { stream: AsyncIterableIterator; offsetMS: OffsetMS }[]
) {
try {
// wait for all results to resolve
const results = await Promise.all(iterators.map(nextWithIndex))
let aliveIteratorsCount = results.length
do {
// if we're dealing with historical data replay
// and need to return combined messages iterable sorted by local timestamp in ascending order
// find resolved one that is the 'oldest'
const oldestResult = results.reduce(findOldestResult, results[0])
const { result, index } = oldestResult
if (result.done) {
aliveIteratorsCount--
// we don't want finished iterators to every be considered 'oldest' again
// hence provide them with result that has local timestamp set to DATE_MAX
// and that is not done
results[index].result = {
done: false,
value: {
localTimestamp: DATE_MAX
}
}
} else {
// yield oldest value and replace with next value from iterable for given index
yield result.value
results[index] = await nextWithIndex(iterators[index], index)
}
} while (aliveIteratorsCount > 0)
} finally {
for (let iterator of iterators) {
;(iterator as any).return()
}
}
}
================================================
FILE: src/computable/booksnapshot.ts
================================================
import { decimalPlaces } from '../handy.ts'
import { OrderBook, OnLevelRemovedCB } from '../orderbook.ts'
import { BookChange, BookPriceLevel, BookSnapshot, Optional } from '../types.ts'
import { Computable } from './computable.ts'
type BookSnapshotComputableOptions = {
name?: string
depth: number
grouping?: number
interval: number
removeCrossedLevels?: boolean
onCrossedLevelRemoved?: OnLevelRemovedCB
}
export const computeBookSnapshots =
(options: BookSnapshotComputableOptions): (() => Computable) =>
() =>
new BookSnapshotComputable(options)
const emptyBookLevel = {
price: undefined,
amount: undefined
}
const levelsChanged = (level1: Optional, level2: Optional) => {
if (level1.amount !== level2.amount) {
return true
}
if (level1.price !== level2.price) {
return true
}
return false
}
class BookSnapshotComputable implements Computable {
public readonly sourceDataTypes = ['book_change']
private _bookChanged = false
private _initialized = false
private readonly _type = 'book_snapshot'
private readonly _orderBook: OrderBook
private readonly _depth: number
private readonly _interval: number
private readonly _name: string
private readonly _grouping: number | undefined
private readonly _groupingDecimalPlaces: number | undefined
private _lastUpdateTimestamp: Date = new Date(-1)
private _bids: Optional[] = []
private _asks: Optional[] = []
constructor({ depth, name, interval, removeCrossedLevels, grouping, onCrossedLevelRemoved }: BookSnapshotComputableOptions) {
this._depth = depth
this._interval = interval
this._grouping = grouping
this._groupingDecimalPlaces = this._grouping ? decimalPlaces(this._grouping) : undefined
this._orderBook = new OrderBook({
removeCrossedLevels,
onCrossedLevelRemoved
})
// initialize all bids/asks levels to empty ones
for (let i = 0; i < this._depth; i++) {
this._bids[i] = emptyBookLevel
this._asks[i] = emptyBookLevel
}
if (name === undefined) {
this._name = `${this._type}_${depth}${this._grouping ? `_grouped${this._grouping}` : ''}_${interval}ms`
} else {
this._name = name
}
}
public *compute(bookChange: BookChange) {
if (this._hasNewSnapshot(bookChange.timestamp)) {
yield this._getSnapshot(bookChange)
}
this._update(bookChange)
// check again after the update as book snapshot with interval set to 0 (real-time) could have changed
// or it's initial snapshot
if (this._hasNewSnapshot(bookChange.timestamp)) {
yield this._getSnapshot(bookChange)
if (this._initialized === false) {
this._initialized = true
}
}
}
public _hasNewSnapshot(timestamp: Date): boolean {
if (this._bookChanged === false) {
return false
}
// report new snapshot anytime book changed
if (this._interval === 0) {
return true
}
// report new snapshot for book snapshots with interval for initial snapshot
if (this._initialized === false) {
return true
}
const currentTimestampTimeBucket = this._getTimeBucket(timestamp)
const snapshotTimestampBucket = this._getTimeBucket(this._lastUpdateTimestamp)
if (currentTimestampTimeBucket > snapshotTimestampBucket) {
// set timestamp to end of snapshot 'interval' period
this._lastUpdateTimestamp = new Date((snapshotTimestampBucket + 1) * this._interval)
return true
}
return false
}
public _update(bookChange: BookChange) {
this._orderBook.update(bookChange)
if (this._grouping !== undefined) {
this._updateSideGrouped(this._orderBook.bids(), this._bids, this._getGroupedPriceForBids)
this._updateSideGrouped(this._orderBook.asks(), this._asks, this._getGroupedPriceForAsks)
} else {
this._updatedNotGrouped()
}
this._lastUpdateTimestamp = bookChange.timestamp
}
private _updatedNotGrouped() {
const bidsIterable = this._orderBook.bids()
const asksIterable = this._orderBook.asks()
for (let i = 0; i < this._depth; i++) {
const bidLevelResult = bidsIterable.next()
const newBid = bidLevelResult.done ? emptyBookLevel : bidLevelResult.value
if (levelsChanged(this._bids[i], newBid)) {
this._bids[i] = { ...newBid }
this._bookChanged = true
}
const askLevelResult = asksIterable.next()
const newAsk = askLevelResult.done ? emptyBookLevel : askLevelResult.value
if (levelsChanged(this._asks[i], newAsk)) {
this._asks[i] = { ...newAsk }
this._bookChanged = true
}
}
}
private _getGroupedPriceForBids = (price: number) => {
const pow = Math.pow(10, this._groupingDecimalPlaces!)
const pricePow = price * pow
const groupPow = this._grouping! * pow
const remainder = (pricePow % groupPow) / pow
return (pricePow - remainder * pow) / pow
}
private _getGroupedPriceForAsks = (price: number) => {
const pow = Math.pow(10, this._groupingDecimalPlaces!)
const pricePow = price * pow
const groupPow = this._grouping! * pow
const remainder = (pricePow % groupPow) / pow
return (pricePow - remainder * pow + (remainder > 0 ? groupPow : 0)) / pow
}
private _updateSideGrouped(
newLevels: IterableIterator,
existingGroupedLevels: Optional[],
getGroupedPriceForLevel: (price: number) => number
) {
let currentGroupedPrice: number | undefined = undefined
let aggAmount = 0
let currentDepth = 0
for (const notGroupedLevel of newLevels) {
const groupedPrice = getGroupedPriceForLevel(notGroupedLevel.price)
if (currentGroupedPrice == undefined) {
currentGroupedPrice = groupedPrice
}
if (currentGroupedPrice != groupedPrice) {
const groupedLevel = {
price: currentGroupedPrice,
amount: aggAmount
}
if (levelsChanged(existingGroupedLevels[currentDepth], groupedLevel)) {
existingGroupedLevels[currentDepth] = groupedLevel
this._bookChanged = true
}
currentDepth++
if (currentDepth === this._depth) {
break
}
currentGroupedPrice = groupedPrice
aggAmount = 0
}
aggAmount += notGroupedLevel.amount
}
if (currentDepth < this._depth && aggAmount > 0) {
const groupedLevel = {
price: currentGroupedPrice,
amount: aggAmount
}
if (levelsChanged(existingGroupedLevels[currentDepth], groupedLevel)) {
existingGroupedLevels[currentDepth] = groupedLevel
this._bookChanged = true
}
}
}
public _getSnapshot(bookChange: BookChange) {
const snapshot: BookSnapshot = {
type: this._type as any,
symbol: bookChange.symbol,
exchange: bookChange.exchange,
name: this._name,
depth: this._depth,
interval: this._interval,
grouping: this._grouping,
bids: [...this._bids],
asks: [...this._asks],
timestamp: this._lastUpdateTimestamp,
localTimestamp: bookChange.localTimestamp
}
this._bookChanged = false
return snapshot
}
private _getTimeBucket(timestamp: Date) {
return Math.floor(timestamp.valueOf() / this._interval)
}
}
================================================
FILE: src/computable/computable.ts
================================================
import { Disconnect, Exchange, NormalizedData } from '../types.ts'
export type Computable = {
readonly sourceDataTypes: string[]
compute(message: NormalizedData): IterableIterator
}
export type ComputableFactory = () => Computable
async function* _compute[], U extends NormalizedData | Disconnect>(
messages: AsyncIterableIterator,
...computables: T
): AsyncIterableIterator[] ? (U extends Disconnect ? U | Z | Disconnect : U | Z) : never> {
try {
const factory = new Computables(computables)
for await (const message of messages) {
// always pass through source message
yield message as any
if (message.type === 'disconnect') {
// reset all computables for given exchange if we've received disconnect for it
factory.reset(message.exchange)
continue
}
const normalizedMessage = message as NormalizedData
const id = normalizedMessage.name !== undefined ? `${normalizedMessage.symbol}:${normalizedMessage.name}` : normalizedMessage.symbol
const computablesMap = factory.getOrCreate(normalizedMessage.exchange, id)
const computables = computablesMap[normalizedMessage.type]
if (!computables) continue
for (const computable of computables) {
for (const computedMessage of computable.compute(normalizedMessage)) {
yield computedMessage
}
}
}
} finally {
messages.return!()
}
}
export function compute[], U extends NormalizedData | Disconnect>(
messages: AsyncIterableIterator,
...computables: T
): AsyncIterableIterator[] ? (U extends Disconnect ? U | Z | Disconnect : U | Z) : never> {
let _iterator = _compute(messages, ...computables)
if ((messages as any).__realtime__ === true) {
;(_iterator as any).__realtime__ = true
}
return _iterator
}
class Computables {
private _computables: {
[ex in Exchange]?: {
[key: string]: { [dataType: string]: Computable[] }
}
} = {}
constructor(private readonly _computablesFactories: ComputableFactory[]) {}
getOrCreate(exchange: Exchange, id: string) {
if (this._computables[exchange] === undefined) {
this._computables[exchange] = {}
}
if (this._computables[exchange]![id] === undefined) {
this._computables[exchange]![id] = createComputablesMap(this._computablesFactories.map((c) => c()))
}
return this._computables[exchange]![id]!
}
reset(exchange: Exchange) {
this._computables[exchange] = undefined
}
}
function createComputablesMap(computables: Computable[]) {
return computables.reduce((acc, computable) => {
computable.sourceDataTypes.forEach((dataType) => {
const existing = acc[dataType]
if (!existing) {
acc[dataType] = [computable]
} else {
acc[dataType].push(computable)
}
})
return acc
}, {} as { [dataType: string]: Computable[] })
}
================================================
FILE: src/computable/index.ts
================================================
export * from './booksnapshot.ts'
export * from './computable.ts'
export * from './tradebar.ts'
================================================
FILE: src/computable/tradebar.ts
================================================
import { BookChange, NormalizedData, Trade, TradeBar, Writeable } from '../types.ts'
import { Computable } from './computable.ts'
const DATE_MIN = new Date(-1)
type BarKind = 'time' | 'volume' | 'tick'
type TradeBarComputableOptions = { kind: BarKind; interval: number; name?: string }
export const computeTradeBars =
(options: TradeBarComputableOptions): (() => Computable) =>
() =>
new TradeBarComputable(options)
const kindSuffix: { [key in BarKind]: string } = {
tick: 'ticks',
time: 'ms',
volume: 'vol'
}
class TradeBarComputable implements Computable {
// use book_change messages as workaround for issue when time passes for new bar to be produced but there's no trades,
// so logic `compute` would not execute
// assumption is that if one subscribes to book changes too then there's pretty good chance that
// even if there are no trades, there's plenty of book changes that trigger computing new trade bar if time passess
public readonly sourceDataTypes = ['trade']
private _inProgressBar: Writeable
private readonly _kind: BarKind
private readonly _interval: number
private readonly _name: string
private readonly _type = 'trade_bar'
constructor({ kind, interval, name }: TradeBarComputableOptions) {
this._kind = kind
this._interval = interval
if (name === undefined) {
this._name = `${this._type}_${interval}${kindSuffix[kind]}`
} else {
this._name = name
}
this._inProgressBar = {} as any
this._reset()
}
public *compute(message: Trade | BookChange) {
// first check if there is a new trade bar for new timestamp for time based trade bars
if (this._hasNewBar(message.timestamp)) {
yield this._computeBar(message)
}
if (message.type !== 'trade') {
return
}
// update in progress trade bar with new data
this._update(message)
// and check again if there is a new trade bar after the update (volume/tick based trade bars)
if (this._hasNewBar(message.timestamp)) {
yield this._computeBar(message)
}
}
private _computeBar(message: NormalizedData) {
this._inProgressBar.localTimestamp = message.localTimestamp
this._inProgressBar.symbol = message.symbol
this._inProgressBar.exchange = message.exchange
const tradeBar: TradeBar = { ...this._inProgressBar }
this._reset()
return tradeBar
}
private _hasNewBar(timestamp: Date): boolean {
// privided timestamp is an exchange trade timestamp in that case
// we bucket based on exchange timestamps when bucketing by time not by localTimestamp
if (this._inProgressBar.trades === 0) {
return false
}
if (this._kind === 'time') {
// TODO: push initial bar at a start?
const currentTimestampTimeBucket = this._getTimeBucket(timestamp)
const openTimestampTimeBucket = this._getTimeBucket(this._inProgressBar.openTimestamp)
if (currentTimestampTimeBucket > openTimestampTimeBucket) {
// set the timestamp to the end of the period of given bucket
this._inProgressBar.timestamp = new Date((openTimestampTimeBucket + 1) * this._interval)
return true
}
return false
}
if (this._kind === 'volume') {
return this._inProgressBar.volume >= this._interval
}
if (this._kind === 'tick') {
return this._inProgressBar.trades >= this._interval
}
return false
}
private _update(trade: Trade) {
const inProgressBar = this._inProgressBar
const isNotOpenedYet = inProgressBar.trades === 0
// some exchanges (like dydx) sometimes publish trade data out of order (older trades published after newer ones)
const tradeIsInCorrectTimeOrder = trade.timestamp.valueOf() >= inProgressBar.timestamp.valueOf()
if (isNotOpenedYet) {
inProgressBar.open = trade.price
inProgressBar.openTimestamp = trade.timestamp
}
if (inProgressBar.high < trade.price) {
inProgressBar.high = trade.price
}
if (inProgressBar.low > trade.price) {
inProgressBar.low = trade.price
}
if (tradeIsInCorrectTimeOrder) {
inProgressBar.close = trade.price
inProgressBar.closeTimestamp = trade.timestamp
inProgressBar.timestamp = trade.timestamp
}
inProgressBar.buyVolume += trade.side === 'buy' ? trade.amount : 0
inProgressBar.sellVolume += trade.side === 'sell' ? trade.amount : 0
inProgressBar.trades += 1
inProgressBar.vwap = (inProgressBar.vwap * inProgressBar.volume + trade.price * trade.amount) / (inProgressBar.volume + trade.amount)
// volume needs to be updated after vwap otherwise vwap calc will go wrong
inProgressBar.volume += trade.amount
}
private _reset() {
const barToReset = this._inProgressBar
barToReset.type = this._type
barToReset.symbol = ''
barToReset.exchange = '' as any
barToReset.name = this._name
barToReset.interval = this._interval
barToReset.kind = this._kind
barToReset.open = 0
barToReset.high = Number.MIN_SAFE_INTEGER
barToReset.low = Number.MAX_SAFE_INTEGER
barToReset.close = 0
barToReset.volume = 0
barToReset.buyVolume = 0
barToReset.sellVolume = 0
barToReset.trades = 0
barToReset.vwap = 0
barToReset.openTimestamp = DATE_MIN
barToReset.closeTimestamp = DATE_MIN
barToReset.localTimestamp = DATE_MIN
barToReset.timestamp = DATE_MIN
}
private _getTimeBucket(timestamp: Date) {
return Math.floor(timestamp.valueOf() / this._interval)
}
}
================================================
FILE: src/consts.ts
================================================
export const EXCHANGES = [
'bitmex',
'deribit',
'binance-futures',
'binance-delivery',
'binance-european-options',
'binance',
'ftx',
'okex-futures',
'okex-options',
'okex-swap',
'okex',
'okex-spreads',
'huobi-dm',
'huobi-dm-swap',
'huobi-dm-linear-swap',
'huobi',
'bitfinex-derivatives',
'bitfinex',
'coinbase',
'coinbase-international',
'cryptofacilities',
'kraken',
'bitstamp',
'gemini',
'poloniex',
'bybit',
'bybit-spot',
'bybit-options',
'phemex',
'delta',
'ftx-us',
'binance-us',
'gate-io-futures',
'gate-io',
'okcoin',
'bitflyer',
'hitbtc',
'coinflex',
'binance-jersey',
'binance-dex',
'upbit',
'ascendex',
'dydx',
'dydx-v4',
'serum',
'mango',
'huobi-dm-options',
'star-atlas',
'crypto-com',
'kucoin',
'kucoin-futures',
'bitnomial',
'woo-x',
'blockchain-com',
'bitget',
'bitget-futures',
'hyperliquid'
] as const
const BINANCE_CHANNELS = ['trade', 'aggTrade', 'ticker', 'depth', 'depthSnapshot', 'bookTicker', 'recentTrades', 'borrowInterest'] as const
const BINANCE_DEX_CHANNELS = ['trades', 'marketDiff', 'depthSnapshot', 'ticker'] as const
const BITFINEX_CHANNELS = ['trades', 'book', 'raw_book', 'ticker'] as const
const BITMEX_CHANNELS = [
'trade',
'orderBookL2',
'liquidation',
'connected',
'announcement',
'chat',
'publicNotifications',
'instrument',
'settlement',
'funding',
'insurance',
'orderBookL2_25',
'orderBook10',
'quote',
'quoteBin1m',
'quoteBin5m',
'quoteBin1h',
'quoteBin1d',
'tradeBin1m',
'tradeBin5m',
'tradeBin1h',
'tradeBin1d'
] as const
const BITSTAMP_CHANNELS = ['live_trades', 'live_orders', 'diff_order_book'] as const
const COINBASE_CHANNELS = [
'match',
'subscriptions',
'received',
'open',
'done',
'change',
'l2update',
'ticker',
'snapshot',
'last_match',
'full_snapshot',
'rfq_matches'
] as const
const DERIBIT_CHANNELS = [
'book',
'deribit_price_index',
'deribit_price_ranking',
'deribit_volatility_index',
'estimated_expiration_price',
'markprice.options',
'perpetual',
'trades',
'ticker',
'quote',
'platform_state',
'instrument.state.any'
] as const
const KRAKEN_CHANNELS = ['trade', 'ticker', 'book', 'spread'] as const
const OKEX_CHANNELS = [
'spot/trade',
'spot/depth',
'spot/depth_l2_tbt',
'spot/ticker',
'system/status',
'margin/interest_rate',
// v5
'trades',
'trades-all',
'books-l2-tbt',
'bbo-tbt',
'books',
'tickers',
'interest-rate-loan-quota',
'vip-interest-rate-loan-quota',
'status',
'instruments',
'taker-volume',
'public-struc-block-trades',
'liquidations',
'loan-ratio',
'public-block-trades'
] as const
const OKCOIN_CHANNELS = [
'spot/trade',
'spot/depth',
'spot/depth_l2_tbt',
'spot/ticker',
'system/status',
'trades',
'books',
'bbo-tbt',
'tickers'
] as const
const OKEX_FUTURES_CHANNELS = [
'futures/trade',
'futures/depth',
'futures/depth_l2_tbt',
'futures/ticker',
'futures/mark_price',
'futures/liquidation',
'index/ticker',
'system/status',
'information/sentiment',
'information/long_short_ratio',
'information/margin',
// v5
'trades',
'trades-all',
'books-l2-tbt',
'bbo-tbt',
'books',
'tickers',
'open-interest',
'mark-price',
'price-limit',
'status',
'instruments',
'index-tickers',
'long-short-account-ratio',
'taker-volume',
'liquidations',
'public-struc-block-trades',
'liquidation-orders',
'estimated-price',
'long-short-account-ratio-contract',
'long-short-account-ratio-contract-top-trader',
'long-short-position-ratio-contract-top-trader',
'public-block-trades',
'taker-volume-contract'
] as const
const OKEX_SWAP_CHANNELS = [
'swap/trade',
'swap/depth',
'swap/depth_l2_tbt',
'swap/ticker',
'swap/funding_rate',
'swap/mark_price',
'swap/liquidation',
'index/ticker',
'system/status',
'information/sentiment',
'information/long_short_ratio',
'information/margin',
//v5
'trades',
'trades-all',
'books-l2-tbt',
'bbo-tbt',
'books',
'tickers',
'open-interest',
'mark-price',
'price-limit',
'funding-rate',
'status',
'instruments',
'index-tickers',
'long-short-account-ratio',
'taker-volume',
'liquidations',
'public-struc-block-trades',
'liquidation-orders',
'long-short-account-ratio-contract',
'long-short-account-ratio-contract-top-trader',
'long-short-position-ratio-contract-top-trader',
'public-block-trades',
'taker-volume-contract'
] as const
const OKEX_OPTIONS_CHANNELS = [
'option/trade',
'option/depth',
'option/depth_l2_tbt',
'option/ticker',
'option/summary',
'option/instruments',
'index/ticker',
'system/status',
'option/trades',
//v5
'trades',
'trades-all',
'books-l2-tbt',
'bbo-tbt',
'books',
'tickers',
'opt-summary',
'status',
'instruments',
'index-tickers',
'open-interest',
'mark-price',
'price-limit',
'public-struc-block-trades',
'option-trades',
'estimated-price',
'public-block-trades'
] as const
const COINFLEX_CHANNELS = ['futures/depth', 'trade', 'ticker'] as const
const CRYPTOFACILITIES_CHANNELS = ['trade', 'trade_snapshot', 'book', 'book_snapshot', 'ticker', 'heartbeat'] as const
const FTX_CHANNELS = [
'orderbook',
'trades',
'instrument',
'markets',
'orderbookGrouped',
'lendingRate',
'borrowRate',
'borrowSummary',
'ticker',
'leveragedTokenInfo',
'busy'
] as const
const GEMINI_CHANNELS = ['trade', 'l2_updates', 'auction_open', 'auction_indicative', 'auction_result'] as const
const BITFLYER_CHANNELS = ['lightning_executions', 'lightning_board_snapshot', 'lightning_board', 'lightning_ticker'] as const
const BINANCE_FUTURES_CHANNELS = [
'trade',
'aggTrade',
'ticker',
'depth',
'markPrice',
'premiumIndex',
'depthSnapshot',
'bookTicker',
'forceOrder',
'openInterest',
'fundingInfo',
'insuranceBalance',
'recentTrades',
'compositeIndex',
'topLongShortAccountRatio',
'topLongShortPositionRatio',
'globalLongShortAccountRatio',
'takerlongshortRatio',
'!contractInfo',
'assetIndex'
] as const
const BINANCE_DELIVERY_CHANNELS = [
'trade',
'aggTrade',
'ticker',
'depth',
'markPrice',
'indexPrice',
'depthSnapshot',
'bookTicker',
'forceOrder',
'openInterest',
'fundingInfo',
'recentTrades',
'topLongShortAccountRatio',
'topLongShortPositionRatio',
'globalLongShortAccountRatio',
'takerBuySellVol',
'!contractInfo'
] as const
const BITFINEX_DERIV_CHANNELS = ['trades', 'book', 'raw_book', 'status', 'liquidations', 'ticker'] as const
const HUOBI_CHANNELS = ['depth', 'detail', 'trade', 'bbo', 'mbp', 'etp', 'mbp.20'] as const
const HUOBI_DM_CHANNELS = [
'depth',
'detail',
'trade',
'bbo',
'basis',
'liquidation_orders',
'contract_info',
'open_interest',
'elite_account_ratio',
'elite_position_ratio'
] as const
const HUOBI_DM_SWAP_CHANNELS = [
'depth',
'detail',
'trade',
'bbo',
'basis',
'funding_rate',
'liquidation_orders',
'contract_info',
'open_interest',
'elite_account_ratio',
'elite_position_ratio'
] as const
const HUOBI_DM_LINEAR_SWAP_CHANNELS = [
'depth',
'detail',
'trade',
'bbo',
'basis',
'funding_rate',
'liquidation_orders',
'contract_info',
'open_interest',
'elite_account_ratio',
'elite_position_ratio'
] as const
const PHEMEX_CHANNELS = ['book', 'orderbook_p', 'trades', 'trades_p', 'market24h', 'spot_market24h', 'perp_market24h_pack_p'] as const
const BYBIT_CHANNELS = [
'trade',
'instrument_info',
'orderBookL2_25',
'insurance',
'orderBook_200',
'liquidation',
'trade',
'instrument_info',
'orderBookL2_25',
'insurance',
'orderBook_200',
'liquidation',
'long_short_ratio',
'orderbook.1',
'orderbook.50',
'orderbook.500',
'publicTrade',
'tickers',
'liquidation',
'allLiquidation',
'orderbook.1000'
] as const
const BYBIT_OPTIONS_CHANNELS = ['orderbook.25', 'orderbook.100', 'publicTrade', 'tickers']
const HITBTC_CHANNELS = ['updateTrades', 'snapshotTrades', 'snapshotOrderbook', 'updateOrderbook'] as const
const FTX_US_CHANNELS = ['orderbook', 'trades', 'markets', 'orderbookGrouped', 'ticker'] as const
const DELTA_CHANNELS = [
'l2_orderbook',
'recent_trade',
'recent_trade_snapshot',
'mark_price',
'spot_price',
'funding_rate',
'product_updates',
'announcements',
'all_trades',
'v2/ticker',
'l1_orderbook',
'l2_updates',
'spot_30mtwap_price'
] as const
const GATE_IO_CHANNELS = ['trades', 'depth', 'ticker', 'book_ticker', 'order_book_update', 'obu'] as const
const GATE_IO_FUTURES_CHANNELS = ['trades', 'order_book', 'tickers', 'book_ticker'] as const
const POLONIEX_CHANNELS = ['price_aggregated_book', 'trades', 'ticker', 'book_lv2'] as const
const UPBIT_CHANNELS = ['trade', 'orderbook', 'ticker'] as const
const ASCENDEX_CHANNELS = ['trades', 'depth-realtime', 'depth-snapshot-realtime', 'bbo', 'futures-pricing-data'] as const
const DYDX_CHANNELS = ['v3_trades', 'v3_orderbook', 'v3_markets'] as const
const DYDX_V4_CHANNELS = ['v4_trades', 'v4_orderbook', 'v4_markets'] as const
const SERUM_CHANNELS = [
'recent_trades',
'trade',
'quote',
'l2snapshot',
'l2update',
'l3snapshot',
'open',
'fill',
'change',
'done'
] as const
const MANGO_CHANNELS = [
'recent_trades',
'trade',
'quote',
'l2snapshot',
'l2update',
'l3snapshot',
'open',
'fill',
'change',
'done'
] as const
const HUOBI_DM_OPTIONS_CHANNELS = ['trade', 'detail', 'depth', 'bbo', 'open_interest', 'option_market_index', 'option_index'] as const
const BYBIT_SPOT_CHANNELS = ['trade', 'bookTicker', 'depth', 'orderbook.1', 'orderbook.50', 'publicTrade', 'tickers', 'lt', 'orderbook.200']
const CRYPTO_COM_CHANNELS = ['trade', 'book', 'ticker', 'settlement', 'index', 'mark', 'funding', 'estimatedfunding']
const KUCOIN_CHANNELS = ['market/ticker', 'market/snapshot', 'market/level2', 'market/match', 'market/level2Snapshot']
const BITNOMIAL_CHANNELS = ['trade', 'level', 'book', 'block', 'status']
const WOOX_CHANNELS = [
'trade',
'orderbook',
'orderbookupdate',
'ticker',
'bbo',
'indexprice',
'markprice',
'openinterest',
'estfundingrate'
]
const BLOCKCHAIN_COM_CHANNELS = ['trades', 'l2', 'l3', 'ticker']
const BINANCE_EUROPEAN_OPTIONS_CHANNELS = [
'trade',
'depth100',
'index',
'markPrice',
'ticker',
'openInterest',
'optionTrade',
'optionTicker',
'depth20',
'bookTicker',
'optionIndexPrice',
'optionMarkPrice',
'optionOpenInterest',
'!optionSymbol'
]
const OKEX_SPREADS_CHANNELS = ['sprd-public-trades', 'sprd-bbo-tbt', 'sprd-books5', 'sprd-tickers']
const KUCOIN_FUTURES_CHANNELS = [
'contractMarket/execution',
'contractMarket/level2',
'contractMarket/level2Snapshot',
'contractMarket/tickerV2',
'contract/instrument',
'contract/details',
'contractMarket/snapshot'
]
const BITGET_CHANNELS = ['trade', 'books1', 'books15']
const BITGET_FUTURES_CHANNELS = ['trade', 'books1', 'books15', 'ticker']
const COINBASE_INTERNATIONAL_CHANNELS = ['INSTRUMENTS', 'MATCH', 'FUNDING', 'RISK', 'LEVEL1', 'LEVEL2', 'CANDLES_ONE_MINUTE']
const HYPERLIQUID_CHANNELS = ['l2Book', 'trades', 'activeAssetCtx', 'activeSpotAssetCtx', 'bbo']
export const EXCHANGE_CHANNELS_INFO = {
bitmex: BITMEX_CHANNELS,
coinbase: COINBASE_CHANNELS,
'coinbase-international': COINBASE_INTERNATIONAL_CHANNELS,
deribit: DERIBIT_CHANNELS,
cryptofacilities: CRYPTOFACILITIES_CHANNELS,
bitstamp: BITSTAMP_CHANNELS,
kraken: KRAKEN_CHANNELS,
okex: OKEX_CHANNELS,
'okex-swap': OKEX_SWAP_CHANNELS,
'okex-futures': OKEX_FUTURES_CHANNELS,
'okex-options': OKEX_OPTIONS_CHANNELS,
binance: BINANCE_CHANNELS,
'binance-jersey': BINANCE_CHANNELS,
'binance-dex': BINANCE_DEX_CHANNELS,
'binance-us': BINANCE_CHANNELS,
bitfinex: BITFINEX_CHANNELS,
ftx: FTX_CHANNELS,
'ftx-us': FTX_US_CHANNELS,
gemini: GEMINI_CHANNELS,
bitflyer: BITFLYER_CHANNELS,
'binance-futures': BINANCE_FUTURES_CHANNELS,
'binance-delivery': BINANCE_DELIVERY_CHANNELS,
'bitfinex-derivatives': BITFINEX_DERIV_CHANNELS,
huobi: HUOBI_CHANNELS,
'huobi-dm': HUOBI_DM_CHANNELS,
'huobi-dm-swap': HUOBI_DM_SWAP_CHANNELS,
'huobi-dm-linear-swap': HUOBI_DM_LINEAR_SWAP_CHANNELS,
bybit: BYBIT_CHANNELS,
'bybit-spot': BYBIT_SPOT_CHANNELS,
'bybit-options': BYBIT_OPTIONS_CHANNELS,
okcoin: OKCOIN_CHANNELS,
hitbtc: HITBTC_CHANNELS,
coinflex: COINFLEX_CHANNELS,
phemex: PHEMEX_CHANNELS,
delta: DELTA_CHANNELS,
'gate-io': GATE_IO_CHANNELS,
'gate-io-futures': GATE_IO_FUTURES_CHANNELS,
poloniex: POLONIEX_CHANNELS,
upbit: UPBIT_CHANNELS,
ascendex: ASCENDEX_CHANNELS,
dydx: DYDX_CHANNELS,
'dydx-v4': DYDX_V4_CHANNELS,
serum: SERUM_CHANNELS,
'star-atlas': SERUM_CHANNELS,
'huobi-dm-options': HUOBI_DM_OPTIONS_CHANNELS,
mango: MANGO_CHANNELS,
'crypto-com': CRYPTO_COM_CHANNELS,
kucoin: KUCOIN_CHANNELS,
bitnomial: BITNOMIAL_CHANNELS,
'woo-x': WOOX_CHANNELS,
'blockchain-com': BLOCKCHAIN_COM_CHANNELS,
'binance-european-options': BINANCE_EUROPEAN_OPTIONS_CHANNELS,
'okex-spreads': OKEX_SPREADS_CHANNELS,
'kucoin-futures': KUCOIN_FUTURES_CHANNELS,
bitget: BITGET_CHANNELS,
'bitget-futures': BITGET_FUTURES_CHANNELS,
hyperliquid: HYPERLIQUID_CHANNELS
}
================================================
FILE: src/debug.ts
================================================
import dbg from 'debug'
export const debug = dbg('tardis-dev')
================================================
FILE: src/downloaddatasets.ts
================================================
import { existsSync } from 'node:fs'
import pMap from 'p-map'
import { debug } from './debug.ts'
import { DatasetType } from './exchangedetails.ts'
import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from './handy.ts'
import { getOptions } from './options.ts'
import { Exchange } from './types.ts'
const CONCURRENCY_LIMIT = 20
const MILLISECONDS_IN_SINGLE_DAY = 24 * 60 * 60 * 1000
const DEFAULT_DOWNLOAD_DIR = './datasets'
const options = getOptions()
export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasetsOptions) {
const { exchange, dataTypes, from, to, symbols } = downloadDatasetsOptions
const apiKey = downloadDatasetsOptions.apiKey !== undefined ? downloadDatasetsOptions.apiKey : options.apiKey
const downloadDir = downloadDatasetsOptions.downloadDir !== undefined ? downloadDatasetsOptions.downloadDir : DEFAULT_DOWNLOAD_DIR
const format = downloadDatasetsOptions.format !== undefined ? downloadDatasetsOptions.format : 'csv'
const getFilename = downloadDatasetsOptions.getFilename !== undefined ? downloadDatasetsOptions.getFilename : getFilenameDefault
const skipIfExists = downloadDatasetsOptions.skipIfExists === undefined ? true : downloadDatasetsOptions.skipIfExists
// in case someone provided 'api/exchange' symbol, transform it to symbol that is accepted by datasets API
const datasetsSymbols = symbols.map((s) => s.replace(/\/|:/g, '-').toUpperCase())
for (const symbol of datasetsSymbols) {
for (const dataType of dataTypes) {
const { daysCountToFetch, startDate } = getDownloadDateRange(downloadDatasetsOptions)
const startTimestamp = new Date().valueOf()
debug('dataset download started for %s %s %s from %s to %s', exchange, dataType, symbol, from, to)
if (daysCountToFetch > 1) {
// start with downloading last day of the range, validates is API key has access to the end range of requested data
await downloadDataSet(
getDownloadOptions({
exchange,
symbol,
apiKey,
downloadDir,
dataType,
format,
getFilename,
date: addDays(startDate, daysCountToFetch - 1)
}),
skipIfExists
)
}
// then download the first day of the range, validates is API key has access to the start range of requested data
await downloadDataSet(
getDownloadOptions({
exchange,
symbol,
apiKey,
downloadDir,
dataType,
format,
getFilename,
date: startDate
}),
skipIfExists
)
// download the rest concurrently up to the CONCURRENCY_LIMIT
await pMap(
sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
(offset) =>
downloadDataSet(
getDownloadOptions({
exchange,
symbol,
apiKey,
downloadDir,
dataType,
format,
getFilename,
date: addDays(startDate, offset)
}),
skipIfExists
),
{ concurrency: CONCURRENCY_LIMIT }
)
const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000
debug('dataset download finished for %s %s %s from %s to %s, time: %s seconds', exchange, dataType, symbol, from, to, elapsedSeconds)
}
}
}
async function downloadDataSet(downloadOptions: DownloadOptions, skipIfExists: boolean) {
if (skipIfExists && existsSync(downloadOptions.downloadPath)) {
debug('dataset %s already exists, skipping download', downloadOptions.downloadPath)
return
} else {
return await download(downloadOptions)
}
}
function getDownloadOptions({
apiKey,
exchange,
dataType,
date,
symbol,
format,
downloadDir,
getFilename
}: {
exchange: Exchange
dataType: DatasetType
symbol: string
date: Date
format: string
apiKey: string
downloadDir: string
getFilename: (options: GetFilenameOptions) => string
}): DownloadOptions {
const year = date.getUTCFullYear()
const month = doubleDigit(date.getUTCMonth() + 1)
const day = doubleDigit(date.getUTCDate())
const url = `${options.datasetsEndpoint}/${exchange}/${dataType}/${year}/${month}/${day}/${encodeURIComponent(symbol)}.${format}.gz`
const filename = getFilename({
dataType,
date,
exchange,
format,
symbol
})
const downloadPath = `${downloadDir}/${filename}`
return {
url,
downloadPath,
userAgent: options._userAgent,
apiKey
}
}
type DownloadOptions = Parameters[0]
export function sanitizeForFilename(s: string) {
return s.replace(/[:\\/?*<>|"]/g, '-')
}
function getFilenameDefault({ exchange, dataType, format, date, symbol }: GetFilenameOptions) {
return `${exchange}_${dataType}_${date.toISOString().split('T')[0]}_${sanitizeForFilename(symbol)}.${format}.gz`
}
function getDownloadDateRange({ from, to }: DownloadDatasetsOptions) {
if (!from || isNaN(Date.parse(from))) {
throw new Error(`Invalid "from" argument: ${from}. Please provide valid date string.`)
}
if (!to || isNaN(Date.parse(to))) {
throw new Error(`Invalid "to" argument: ${to}. Please provide valid date string.`)
}
const toDate = parseAsUTCDate(to)
const fromDate = parseAsUTCDate(from)
const daysCountToFetch = Math.floor((toDate.getTime() - fromDate.getTime()) / MILLISECONDS_IN_SINGLE_DAY)
if (daysCountToFetch < 1) {
throw new Error(`Invalid "to" and "from" arguments combination. Please provide "to" day that is later than "from" day.`)
}
return {
startDate: fromDate,
daysCountToFetch
}
}
type GetFilenameOptions = {
exchange: Exchange
dataType: DatasetType
symbol: string
date: Date
format: string
}
type DownloadDatasetsOptions = {
exchange: Exchange
dataTypes: DatasetType[]
symbols: string[]
from: string
to: string
format?: 'csv'
apiKey?: string
downloadDir?: string
getFilename?: (options: GetFilenameOptions) => string
skipIfExists?: boolean
}
================================================
FILE: src/exchangedetails.ts
================================================
import { getJSON } from './handy.ts'
import { getOptions } from './options.ts'
import { Exchange, FilterForExchange } from './types.ts'
export async function getExchangeDetails(exchange: T) {
const options = getOptions()
const { data } = await getJSON(`${options.endpoint}/exchanges/${exchange}`)
return data as ExchangeDetails
}
export type SymbolType = 'spot' | 'future' | 'perpetual' | 'option' | 'combo'
export type DatasetType =
| 'trades'
| 'incremental_book_L2'
| 'quotes'
| 'derivative_ticker'
| 'options_chain'
| 'book_snapshot_25'
| 'book_snapshot_5'
| 'liquidations'
| 'book_ticker'
export type Stats = {
trades: number
bookChanges: number
}
type Datasets = {
formats: ['csv']
exportedFrom: string
exportedUntil: string
stats: Stats
symbols: {
id: string
type: SymbolType
availableSince: string
availableTo?: string
dataTypes: DatasetType[]
}[]
}
type ChannelDetails = {
name: string
description: string
frequency: string
frequencySource: string
exchangeDocsUrl?: string
sourceFor?: string[]
availableSince: string
availableTo?: string
apiVersion?: string
additionalInfo?: string
generated?: true
}
type DataCenter = {
host: string
regionId: string
location: string
}
type DataCollectionDetails = {
recorderDataCenter: DataCenter
recorderDataCenterChanges?: {
until: string
dataCenter: DataCenter
}[]
wssConnection?: {
url: string
apiVersion?: string
proxiedViaCloudflare?: boolean
}
wssConnectionChanges?: {
until: string
url?: string
apiVersion?: string
proxiedViaCloudflare?: boolean
}[]
exchangeDataCenter?: DataCenter
exchangeDataCenterChanges?: {
until: string
dataCenter: DataCenter
}[]
}
export type ExchangeDetailsBase = {
id: T
name: string
enabled: boolean
delisted?: boolean
availableSince: string
availableTo?: string
availableChannels: FilterForExchange[T]['channel'][]
availableSymbols: {
id: string
type: SymbolType
availableSince: string
availableTo?: string
name?: string
}[]
incidentReports: {
from: string
to: string
status: 'resolved' | 'wontfix' | 'unresolved'
details: string
}[]
channelDetails: ChannelDetails[]
apiDocsUrl?: string
dataCollectionDetails?: DataCollectionDetails
datasets: Datasets
}
type ExchangeDetails = ExchangeDetailsBase
================================================
FILE: src/filter.ts
================================================
import { NormalizedData, Disconnect, Trade } from './types.ts'
import { CappedSet } from './handy.ts'
export async function* filter(messages: AsyncIterableIterator, filter: (message: T) => boolean) {
for await (const message of messages) {
if (filter(message)) {
yield message
}
}
}
export function uniqueTradesOnly(
{
maxWindow,
onDuplicateFound,
skipStaleOlderThanSeconds
}: {
maxWindow: number
skipStaleOlderThanSeconds?: number
onDuplicateFound?: (trade: Trade) => void
} = {
maxWindow: 500
}
) {
const perSymbolQueues = {} as {
[key: string]: CappedSet
}
return (message: T) => {
// pass trough any message that is not a trade
if (message.type !== 'trade') {
return true
} else {
const trade = message as unknown as Trade
// pass trough trades that can't be uniquely identified
// ignore index trades
if (trade.id === undefined || trade.symbol.startsWith('.')) {
return true
} else {
let alreadySeenTrades = perSymbolQueues[trade.symbol]
if (alreadySeenTrades === undefined) {
perSymbolQueues[trade.symbol] = new CappedSet(maxWindow)
alreadySeenTrades = perSymbolQueues[trade.symbol]
}
const isDuplicate = alreadySeenTrades.has(trade.id)
const isStale =
skipStaleOlderThanSeconds !== undefined &&
trade.localTimestamp.valueOf() - trade.timestamp.valueOf() > skipStaleOlderThanSeconds * 1000
if (isDuplicate || isStale) {
if (onDuplicateFound !== undefined) {
onDuplicateFound(trade)
}
// refresh duplicated key position so it's added back at the beginning of the queue
alreadySeenTrades.remove(trade.id)
alreadySeenTrades.add(trade.id)
return false
} else {
alreadySeenTrades.add(trade.id)
return true
}
}
}
}
}
================================================
FILE: src/handy.ts
================================================
import crypto, { createHash } from 'crypto'
import { createWriteStream, mkdirSync, rmSync } from 'node:fs'
import { rename } from 'node:fs/promises'
import type { RequestOptions, Agent } from 'https'
import followRedirects from 'follow-redirects'
import * as httpsProxyAgentPkg from 'https-proxy-agent'
import path from 'path'
import { debug } from './debug.ts'
import { Mapper } from './mappers/index.ts'
import { Disconnect, Exchange, Filter, FilterForExchange } from './types.ts'
import * as socksProxyAgentPkg from 'socks-proxy-agent'
const { http, https } = followRedirects
const { HttpsProxyAgent } = httpsProxyAgentPkg
const { SocksProxyAgent } = socksProxyAgentPkg
export function parseAsUTCDate(val: string) {
// Treat date-only and minute-level strings as UTC instead of local time.
if (val.endsWith('Z') === false) {
val += 'Z'
}
const date = new Date(val)
return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours(), date.getUTCMinutes()))
}
export function wait(delayMS: number) {
return new Promise((resolve) => {
setTimeout(resolve, delayMS)
})
}
export function getRandomString() {
return crypto.randomBytes(24).toString('hex')
}
export function formatDateToPath(date: Date) {
const year = date.getUTCFullYear()
const month = doubleDigit(date.getUTCMonth() + 1)
const day = doubleDigit(date.getUTCDate())
const hour = doubleDigit(date.getUTCHours())
const minute = doubleDigit(date.getUTCMinutes())
return `${year}/${month}/${day}/${hour}/${minute}`
}
export function doubleDigit(input: number) {
return input < 10 ? '0' + input : '' + input
}
export function sha256(obj: object) {
return createHash('sha256').update(JSON.stringify(obj)).digest('hex')
}
export function addMinutes(date: Date, minutes: number) {
return new Date(date.getTime() + minutes * 60000)
}
export function addDays(date: Date, days: number) {
return new Date(date.getTime() + days * 60000 * 1440)
}
export function* sequence(end: number, seed = 0) {
let current = seed
while (current < end) {
yield current
current += 1
}
return
}
export const ONE_SEC_IN_MS = 1000
export class HttpError extends Error {
constructor(public readonly status: number, public readonly responseText: string, public readonly url: string) {
super(`HttpError: status code: ${status}, response text: ${responseText}`)
}
}
class HttpClientError extends Error {
constructor(public readonly response: HttpResponse, public readonly method: string, public readonly url: string) {
super(`HTTP ${method} ${url} failed with status ${response.statusCode}`)
}
}
export function* take(iterable: Iterable, length: number) {
if (length === 0) {
return
}
for (const item of iterable) {
yield item
length--
if (length === 0) {
return
}
}
}
export async function* normalizeMessages(
exchange: Exchange,
symbols: string[] | undefined,
messages: AsyncIterableIterator<{ localTimestamp: Date; message: any } | undefined>,
mappers: Mapper[],
createMappers: (localTimestamp: Date) => Mapper[],
withDisconnectMessages: boolean | undefined,
filter?: (symbol: string) => boolean,
currentTimestamp?: Date | undefined
) {
let previousLocalTimestamp: Date | undefined = currentTimestamp
let mappersForExchange: Mapper[] | undefined = mappers
if (mappersForExchange.length === 0) {
throw new Error(`Can't normalize data without any normalizers provided`)
}
for await (const messageWithTimestamp of messages) {
if (messageWithTimestamp === undefined) {
// we received undefined meaning Websocket disconnection
// lets create new mappers with clean state for 'new connection'
mappersForExchange = undefined
// if flag withDisconnectMessages is set, yield disconnect message
if (withDisconnectMessages === true && previousLocalTimestamp !== undefined) {
const disconnect: Disconnect = {
type: 'disconnect',
exchange,
localTimestamp: previousLocalTimestamp,
symbols
}
yield disconnect as any
}
continue
}
if (mappersForExchange === undefined) {
mappersForExchange = createMappers(messageWithTimestamp.localTimestamp)
}
previousLocalTimestamp = messageWithTimestamp.localTimestamp
for (const mapper of mappersForExchange) {
if (mapper.canHandle(messageWithTimestamp.message)) {
const mappedMessages = mapper.map(messageWithTimestamp.message, messageWithTimestamp.localTimestamp)
if (!mappedMessages) {
continue
}
for (const message of mappedMessages) {
if (filter === undefined) {
yield message
} else if (filter(message.symbol)) {
yield message
}
}
}
}
}
}
export function getFilters(mappers: Mapper[], symbols?: string[]) {
const filters = mappers.flatMap((mapper) => mapper.getFilters(symbols))
const deduplicatedFilters = filters.reduce((prev, current) => {
const matchingExisting = prev.find((c) => c.channel === current.channel)
if (matchingExisting !== undefined) {
if (matchingExisting.symbols !== undefined && current.symbols) {
for (let symbol of current.symbols) {
if (matchingExisting.symbols.includes(symbol) === false) {
matchingExisting.symbols.push(symbol)
}
}
} else if (current.symbols) {
matchingExisting.symbols = [...current.symbols]
}
} else {
prev.push(current)
}
return prev
}, [] as FilterForExchange[T][])
return deduplicatedFilters
}
export function* batch(symbols: string[], batchSize: number) {
for (let i = 0; i < symbols.length; i += batchSize) {
yield symbols.slice(i, i + batchSize)
}
}
export function* batchObjects(payload: T[], batchSize: number) {
for (let i = 0; i < payload.length; i += batchSize) {
yield payload.slice(i, i + batchSize)
}
}
export function parseμs(dateString: string): number {
// check if we have ISO 8601 format date string, e.g: 2019-06-01T00:03:03.1238784Z or 2020-07-22T00:09:16.836773Z
// or 2020-03-01T00:00:24.893456+00:00
if (dateString.length === 27 || dateString.length === 28 || dateString.length === 32 || dateString.length === 30) {
return Number(dateString.slice(23, 26))
}
return 0
}
export function optimizeFilters(filters: Filter[]) {
// deduplicate filters (if the channel was provided multiple times)
const optimizedFilters = filters.reduce((prev, current) => {
const matchingExisting = prev.find((c) => c.channel === current.channel)
if (matchingExisting) {
// both previous and current have symbols let's merge them
if (matchingExisting.symbols && current.symbols) {
matchingExisting.symbols.push(...current.symbols)
} else if (current.symbols) {
matchingExisting.symbols = [...current.symbols]
}
} else {
prev.push(current)
}
return prev
}, [] as Filter[])
// sort filters in place to improve local disk cache ratio (no matter filters order if the same filters are provided will hit the cache)
optimizedFilters.sort((f1, f2) => {
if (f1.channel < f2.channel) {
return -1
}
if (f1.channel > f2.channel) {
return 1
}
return 0
})
// sort and deduplicate filters symbols
optimizedFilters.forEach((filter) => {
if (filter.symbols) {
filter.symbols = [...new Set(filter.symbols)].sort()
}
})
return optimizedFilters
}
const httpsAgent = new https.Agent({
keepAlive: true,
keepAliveMsecs: 10 * ONE_SEC_IN_MS,
maxSockets: 120
})
export const httpsProxyAgent: Agent | undefined =
process.env.HTTP_PROXY !== undefined
? new HttpsProxyAgent(process.env.HTTP_PROXY)
: process.env.SOCKS_PROXY !== undefined
? new SocksProxyAgent(process.env.SOCKS_PROXY)
: undefined
const DEFAULT_FETCH_RETRY_LIMIT = 2
type HttpRetryOptions =
| number
| {
limit?: number
statusCodes?: number[]
maxRetryAfter?: number
}
type HttpRequestOptions = {
headers?: Record
body?: string | object
timeout?: number
retry?: HttpRetryOptions
}
type HttpResponse = {
statusCode: number
headers: Record
body: string
}
type JSONResponse = {
data: T
headers: Record
statusCode: number
}
type RetrySettings = {
limit: number
maxRetryAfter?: number
statusCodes?: Set
}
function getRetrySettings(method: string, retry?: HttpRetryOptions): RetrySettings {
const retryOptions = typeof retry === 'object' ? retry : undefined
const retryEnabled = method === 'GET' || retry !== undefined
const limit = typeof retry === 'number' ? retry : retryOptions?.limit ?? (retryEnabled ? DEFAULT_FETCH_RETRY_LIMIT : 0)
return {
limit,
maxRetryAfter: retryOptions?.maxRetryAfter,
statusCodes: retryOptions?.statusCodes ? new Set(retryOptions.statusCodes) : undefined
}
}
function parseResponseHeaders(headers: Headers) {
return Object.fromEntries(headers.entries())
}
function parseNodeResponseHeaders(headers: Record) {
return Object.fromEntries(
Object.entries(headers).flatMap(([key, value]) => {
if (value === undefined) {
return []
}
return [[key.toLowerCase(), Array.isArray(value) ? value.join(', ') : value]]
})
)
}
function createHttpResponse(statusCode: number, headers: Record, body: string): HttpResponse {
return {
statusCode,
headers,
body
}
}
function prepareRequest(method: string, options: HttpRequestOptions) {
if (options.body === undefined) {
return {
headers: options.headers,
body: undefined
}
}
const headers = { ...options.headers }
const body = typeof options.body === 'string' ? options.body : JSON.stringify(options.body)
if (method !== 'GET' && headers['Content-Type'] === undefined && headers['content-type'] === undefined) {
headers['Content-Type'] = 'application/json'
}
return {
headers,
body
}
}
function getRetryAfterDelayMS(headers: Record, maxRetryAfter?: number) {
const retryAfterHeader = headers['retry-after']
if (retryAfterHeader === undefined) {
return
}
const parsedSeconds = Number.parseFloat(retryAfterHeader)
let delayMS: number | undefined
if (Number.isFinite(parsedSeconds)) {
delayMS = parsedSeconds * ONE_SEC_IN_MS
} else {
const parsedDate = Date.parse(retryAfterHeader)
if (Number.isFinite(parsedDate)) {
delayMS = parsedDate - Date.now()
}
}
if (delayMS === undefined || delayMS < 0) {
return
}
if (maxRetryAfter !== undefined && delayMS > maxRetryAfter) {
return
}
return delayMS
}
function getRetryDelayMS(attempt: number, headers: Record, maxRetryAfter?: number) {
const retryAfterDelayMS = getRetryAfterDelayMS(headers, maxRetryAfter)
if (retryAfterDelayMS !== undefined) {
return retryAfterDelayMS
}
return Math.min(250 * 2 ** (attempt - 1), 5000)
}
function isRetryableStatus(statusCode: number, retrySettings: RetrySettings) {
if (retrySettings.statusCodes !== undefined) {
return retrySettings.statusCodes.has(statusCode)
}
return statusCode === 408 || statusCode === 429 || statusCode >= 500
}
function shouldRetryHttpStatus(attempt: number, response: HttpResponse, retrySettings: RetrySettings) {
return attempt <= retrySettings.limit && isRetryableStatus(response.statusCode, retrySettings)
}
function shouldRetryHttpError(attempt: number, retrySettings: RetrySettings) {
return attempt <= retrySettings.limit
}
async function requestViaFetch(method: string, url: string, options: HttpRequestOptions): Promise {
const controller = new AbortController()
const timeoutMS = options.timeout
const timeoutId = timeoutMS !== undefined ? setTimeout(() => controller.abort(), timeoutMS) : undefined
const preparedRequest = prepareRequest(method, options)
try {
const response = await fetch(url, {
method,
headers: preparedRequest.headers,
body: preparedRequest.body,
signal: controller.signal
})
const body = await response.text()
return createHttpResponse(response.status, parseResponseHeaders(response.headers), body)
} catch (error) {
if (controller.signal.aborted) {
throw new Error('Request timed out')
}
throw error
} finally {
if (timeoutId !== undefined) {
clearTimeout(timeoutId)
}
}
}
async function requestViaProxy(method: string, url: string, options: HttpRequestOptions): Promise {
const requestClient = new URL(url).protocol === 'http:' ? http : https
const preparedRequest = prepareRequest(method, options)
return await new Promise((resolve, reject) => {
const request = requestClient
.request(
url,
{
method,
agent: httpsProxyAgent,
headers: preparedRequest.headers,
timeout: options.timeout
},
(response) => {
response.setEncoding('utf8')
let body = ''
response.on('error', reject)
response.on('data', (chunk) => (body += chunk))
response.on('end', () => {
resolve(createHttpResponse(response.statusCode ?? 0, parseNodeResponseHeaders(response.headers), body))
})
}
)
.on('error', reject)
.on('timeout', () => {
reject(new Error('Request timed out'))
request.destroy()
})
if (preparedRequest.body !== undefined) {
request.write(preparedRequest.body)
}
request.end()
})
}
async function request(method: string, url: string, options: HttpRequestOptions = {}) {
const retrySettings = getRetrySettings(method, options.retry)
for (let attempt = 1; ; attempt += 1) {
try {
const response =
httpsProxyAgent === undefined ? await requestViaFetch(method, url, options) : await requestViaProxy(method, url, options)
if (response.statusCode >= 200 && response.statusCode < 300) {
return response
}
if (shouldRetryHttpStatus(attempt, response, retrySettings)) {
await wait(getRetryDelayMS(attempt, response.headers, retrySettings.maxRetryAfter))
continue
}
throw new HttpClientError(response, method, url)
} catch (error) {
if (error instanceof HttpClientError) {
throw error
}
if (shouldRetryHttpError(attempt, retrySettings)) {
await wait(Math.min(250 * 2 ** (attempt - 1), 5000))
continue
}
throw error
}
}
}
async function requestJSON(method: string, url: string, options?: HttpRequestOptions): Promise> {
const response = await request(method, url, options)
return {
data: JSON.parse(response.body) as T,
headers: response.headers,
statusCode: response.statusCode
}
}
export function getJSON(url: string, options?: HttpRequestOptions) {
return requestJSON('GET', url, options)
}
export function postJSON(url: string, options?: HttpRequestOptions) {
return requestJSON('POST', url, options)
}
export async function download({
apiKey,
downloadPath,
url,
userAgent,
appendContentEncodingExtension = false,
acceptEncoding = 'gzip'
}: {
url: string
downloadPath: string
userAgent: string
apiKey: string
appendContentEncodingExtension?: boolean
acceptEncoding?: string
}) {
const httpRequestOptions = {
agent: httpsProxyAgent !== undefined ? httpsProxyAgent : httpsAgent,
timeout: 90 * ONE_SEC_IN_MS,
headers: {
'Accept-Encoding': acceptEncoding,
'User-Agent': userAgent,
Authorization: apiKey ? `Bearer ${apiKey}` : ''
}
}
const MAX_ATTEMPTS = 30
let attempts = 0
while (true) {
// simple retry logic when fetching from the network...
attempts++
try {
const addRetryAttempt = attempts - 1 > 0 && url.endsWith('gz')
if (addRetryAttempt) {
return await _downloadFile(httpRequestOptions, `${url}?retryAttempt=${attempts - 1}`, downloadPath, appendContentEncodingExtension)
} else {
return await _downloadFile(httpRequestOptions, url, downloadPath, appendContentEncodingExtension)
}
} catch (error) {
const unsupportedDataFeedEncoding = error instanceof Error && error.message.startsWith('Unsupported data feed content encoding')
const badOrUnauthorizedRequest =
error instanceof HttpError &&
((error.status === 400 && error.message.includes('ISO 8601 format') === false) || error.status === 401)
const tooManyRequests = error instanceof HttpError && error.status === 429
const internalServiceError = error instanceof HttpError && error.status === 500
// do not retry when we've got bad or unauthorized request or enough attempts
if (unsupportedDataFeedEncoding || badOrUnauthorizedRequest || attempts === MAX_ATTEMPTS) {
throw error
}
const randomIngridient = Math.random() * 500
const attemptsDelayMS = Math.min(Math.pow(2, attempts) * ONE_SEC_IN_MS, 120 * ONE_SEC_IN_MS)
let nextAttemptDelayMS = randomIngridient + attemptsDelayMS
if (tooManyRequests) {
// when too many requests received wait one minute
nextAttemptDelayMS += 60 * ONE_SEC_IN_MS
}
if (internalServiceError) {
nextAttemptDelayMS = nextAttemptDelayMS * 2
}
debug('download file error: %o, next attempt delay: %d, url %s, path: %s', error, nextAttemptDelayMS, url, downloadPath)
await wait(nextAttemptDelayMS)
}
}
}
const tmpFileCleanups = new Map void>()
export function cleanTempFiles() {
tmpFileCleanups.forEach((cleanup) => cleanup())
}
async function _downloadFile(requestOptions: RequestOptions, url: string, downloadPath: string, appendContentEncodingExtension: boolean) {
// first ensure that directory where we want to download file exists
mkdirSync(path.dirname(downloadPath), { recursive: true })
// create write file stream that we'll write data into - first as unconfirmed temp file
const tmpFilePath = `${downloadPath}${crypto.randomBytes(8).toString('hex')}.unconfirmed`
const fileWriteStream = createWriteStream(tmpFilePath)
const cleanup = () => {
try {
fileWriteStream.destroy()
rmSync(tmpFilePath, { force: true })
} catch {}
}
tmpFileCleanups.set(tmpFilePath, cleanup)
let finalDownloadPath = downloadPath
try {
// based on https://github.com/nodejs/node/issues/28172 - only reliable way to consume response stream and avoiding all the 'gotchas'
await new Promise((resolve, reject) => {
const req = https
.get(url, requestOptions, (res) => {
const { statusCode } = res
if (statusCode !== 200) {
// read the error response text and throw it as an HttpError
res.setEncoding('utf8')
let body = ''
res.on('error', reject)
res.on('data', (chunk) => (body += chunk))
res.on('end', () => {
reject(new HttpError(statusCode!, body, url))
})
} else {
if (appendContentEncodingExtension) {
const contentEncoding = asSingleHeaderValue(res.headers['content-encoding'])
if (contentEncoding === 'zstd') {
finalDownloadPath = `${downloadPath}.zst`
} else if (contentEncoding === undefined || contentEncoding === 'gzip') {
finalDownloadPath = `${downloadPath}.gz`
} else {
reject(new Error(`Unsupported data feed content encoding: ${contentEncoding}`))
return
}
}
// consume the response stream by writing it to the file
res
.on('error', reject)
.on('aborted', () => reject(new Error('Request aborted')))
.pipe(fileWriteStream)
.on('error', reject)
.on('finish', () => {
if (res.complete) {
resolve()
} else {
reject(new Error('The connection was terminated while the message was still being sent'))
}
})
}
})
.on('error', reject)
.on('timeout', () => {
debug('download file request timeout, %s', url)
reject(new Error('Request timed out'))
req.destroy()
})
})
// finally when saving from the network to file has succeded, rename tmp file to normal name
// then we're sure that responses is 100% saved and also even if different process was doing the same we're good
await rename(tmpFilePath, finalDownloadPath)
return {
downloadPath: finalDownloadPath
}
} finally {
tmpFileCleanups.delete(tmpFilePath)
cleanup()
}
}
function asSingleHeaderValue(headerValue: string | string[] | undefined) {
if (Array.isArray(headerValue)) {
return headerValue[0]
}
return headerValue
}
export class CircularBuffer {
private _buffer: T[] = []
private _index: number = 0
constructor(private readonly _bufferSize: number) {}
append(value: T) {
const isFull = this._buffer.length === this._bufferSize
let poppedValue
if (isFull) {
poppedValue = this._buffer[this._index]
}
this._buffer[this._index] = value
this._index = (this._index + 1) % this._bufferSize
return poppedValue
}
*items() {
for (let i = 0; i < this._buffer.length; i++) {
const index = (this._index + i) % this._buffer.length
yield this._buffer[index]
}
}
get count() {
return this._buffer.length
}
clear() {
this._buffer = []
this._index = 0
}
}
export class CappedSet {
private _set = new Set()
constructor(private readonly _maxSize: number) {}
public has(value: T) {
return this._set.has(value)
}
public add(value: T) {
if (this._set.size >= this._maxSize) {
this._set.delete(this._set.keys().next().value!)
}
this._set.add(value)
}
public remove(value: T) {
this._set.delete(value)
}
public size() {
return this._set.size
}
}
function hasFraction(n: number) {
return Math.abs(Math.round(n) - n) > 1e-10
}
// https://stackoverflow.com/a/44815797
export function decimalPlaces(n: number) {
let count = 0
// multiply by increasing powers of 10 until the fractional part is ~ 0
while (hasFraction(n * 10 ** count) && isFinite(10 ** count)) count++
return count
}
export function asNumberIfValid(val: string | number | undefined | null) {
if (val === undefined || val === null) {
return
}
var asNumber = Number(val)
if (isNaN(asNumber) || isFinite(asNumber) === false) {
return
}
if (asNumber === 0) {
return
}
return asNumber
}
export function upperCaseSymbols(symbols?: string[]) {
if (symbols !== undefined) {
return symbols.map((s) => s.toUpperCase())
}
return
}
export function lowerCaseSymbols(symbols?: string[]) {
if (symbols !== undefined) {
return symbols.map((s) => s.toLowerCase())
}
return
}
export const fromMicroSecondsToDate = (micros: number) => {
const isMicroseconds = micros > 1e15 // Check if the number is likely in microseconds
if (!isMicroseconds) {
return new Date(micros)
}
const timestamp = new Date(micros / 1000)
timestamp.μs = micros % 1000
return timestamp
}
export function onlyUnique(value: string, index: number, array: string[]) {
return array.indexOf(value) === index
}
================================================
FILE: src/index.ts
================================================
export * from './apikeyaccessinfo.ts'
export * from './clearcache.ts'
export * from './combine.ts'
export * from './computable/index.ts'
export * from './consts.ts'
export * from './exchangedetails.ts'
export * from './mappers/index.ts'
export { init } from './options.ts'
export * from './orderbook.ts'
export * from './realtimefeeds/index.ts'
export * from './replay.ts'
export * from './stream.ts'
export * from './downloaddatasets.ts'
export * from './types.ts'
export * from './filter.ts'
export * from './instrumentinfo.ts'
================================================
FILE: src/instrumentinfo.ts
================================================
import { getOptions } from './options.ts'
import type { SymbolType } from './exchangedetails.ts'
import type { Exchange } from './types.ts'
import { getJSON } from './handy.ts'
export async function getInstrumentInfo(exchange: Exchange): Promise
export async function getInstrumentInfo(exchange: Exchange | Exchange[], filter: InstrumentInfoFilter): Promise
export async function getInstrumentInfo(exchange: Exchange, symbol: string): Promise
export async function getInstrumentInfo(exchange: Exchange | Exchange[], filterOrSymbol?: InstrumentInfoFilter | string) {
if (Array.isArray(exchange)) {
const exchanges = exchange
const results = await Promise.all(exchanges.map((e) => getInstrumentInfoForExchange(e, filterOrSymbol)))
return results.flat()
} else {
return getInstrumentInfoForExchange(exchange, filterOrSymbol)
}
}
async function getInstrumentInfoForExchange(exchange: Exchange, filterOrSymbol?: InstrumentInfoFilter | string) {
const options = getOptions()
let url = `${options.endpoint}/instruments/${exchange}`
if (typeof filterOrSymbol === 'string') {
url += `/${encodeURIComponent(filterOrSymbol)}`
} else if (typeof filterOrSymbol === 'object') {
url += `?filter=${encodeURIComponent(JSON.stringify(filterOrSymbol))}`
}
try {
const { data } = await getJSON(url, {
headers: { Authorization: `Bearer ${options.apiKey}` }
})
return data
} catch (e: any) {
// expose 400 error message from server
if (e.response?.statusCode === 400) {
let err: { code: Number; message: string }
try {
err = JSON.parse(e.response.body)
} catch {
throw e
}
throw err ? new Error(`${err.message} (${err.code})`) : e
} else {
throw e
}
}
}
type InstrumentInfoFilter = {
baseCurrency?: string | string[]
quoteCurrency?: string | string[]
type?: SymbolType | SymbolType[]
contractType?: ContractType | ContractType[]
active?: boolean
}
export type ContractType =
| 'move'
| 'linear_future'
| 'inverse_future'
| 'quanto_future'
| 'linear_perpetual'
| 'inverse_perpetual'
| 'quanto_perpetual'
| 'put_option'
| 'call_option'
| 'turbo_put_option'
| 'turbo_call_option'
| 'spread'
| 'interest_rate_swap'
| 'repo'
| 'index'
export interface InstrumentInfo {
/** symbol id */
id: string
/** dataset symbol id, may differ from id */
datasetId?: string
/** exchange id */
exchange: string
/** normalized, so for example bitmex XBTUSD has base currency set to BTC not XBT */
baseCurrency: string
/** normalized, so for example bitfinex BTCUST has quote currency set to USDT, not UST */
quoteCurrency: string
type: SymbolType
/** derivative contract type */
contractType?: ContractType
/** indicates if the instrument can currently be traded. */
active: boolean
/** date in ISO format */
availableSince: string
/** date in ISO format */
availableTo?: string
/** date in ISO format, when the instrument was first listed on the exchange */
listing?: string
/** in ISO format, only for futures and options */
expiry?: string
/** expiration schedule type */
expirationType?: 'daily' | 'weekly' | 'next_week' | 'quarter' | 'next_quarter'
/** the underlying index for derivatives */
underlyingIndex?: string
/** price tick size, price precision can be calculated from it */
priceIncrement: number
/** amount tick size, amount/size precision can be calculated from it */
amountIncrement: number
/** min order size */
minTradeAmount: number
/** minimum notional value */
minNotional?: number
/** consider it as illustrative only, as it depends in practice on account traded volume levels, different categories, VIP levels, owning exchange currency etc */
makerFee: number
/** consider it as illustrative only, as it depends in practice on account traded volume levels, different categories, VIP levels, owning exchange currency etc */
takerFee: number
/** only for derivatives */
inverse?: boolean
/** only for derivatives */
contractMultiplier?: number
/** only for quanto instruments */
quanto?: boolean
/** only for quanto instruments as settlement currency is different base/quote currency */
settlementCurrency?: string
/** strike price, only for options */
strikePrice?: number
/** option type, only for options */
optionType?: 'call' | 'put'
/** margin mode */
marginMode?: 'isolated' | 'cross'
/** whether margin trading is supported (spot) */
margin?: boolean
/** if this instrument is an alias for another */
aliasFor?: string
/** historical changes to instrument parameters */
changes?: {
until: string
priceIncrement?: number
amountIncrement?: number
contractMultiplier?: number
minTradeAmount?: number
makerFee?: number
takerFee?: number
quanto?: boolean
inverse?: boolean
settlementCurrency?: string
underlyingIndex?: string
contractType?: ContractType
quoteCurrency?: string
type?: string
}[]
}
================================================
FILE: src/mappers/ascendex.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
export class AscendexTradesMapper implements Mapper<'ascendex', Trade> {
canHandle(message: AscendexTrade) {
return message.m === 'trades'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
} as const
]
}
*map(message: AscendexTrade, localTimestamp: Date): IterableIterator {
for (let trade of message.data) {
yield {
type: 'trade',
symbol: message.symbol,
exchange: 'ascendex',
id: undefined,
price: Number(trade.p),
amount: Number(trade.q),
side: trade.bm === true ? 'sell' : 'buy',
timestamp: new Date(trade.ts),
localTimestamp: localTimestamp
}
}
}
}
export class AscendexBookChangeMapper implements Mapper<'ascendex', BookChange> {
canHandle(message: AscendexDepthRealTime | AscendexDepthRealTimeSnapshot) {
return message.m === 'depth-realtime' || message.m === 'depth-snapshot-realtime'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'depth-realtime',
symbols
} as const,
{
channel: 'depth-snapshot-realtime',
symbols
} as const
]
}
*map(message: AscendexDepthRealTime | AscendexDepthRealTimeSnapshot, localTimestamp: Date): IterableIterator {
if (!message.symbol || !message.data.bids || !message.data.asks) {
return
}
yield {
type: 'book_change',
symbol: message.symbol,
exchange: 'ascendex',
isSnapshot: message.m === 'depth-snapshot-realtime',
bids: message.data.bids.map(this.mapBookLevel),
asks: message.data.asks.map(this.mapBookLevel),
timestamp: message.data.ts > 0 ? new Date(message.data.ts) : localTimestamp,
localTimestamp
}
}
protected mapBookLevel(level: AscendexPriceLevel) {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
}
export class AscendexDerivativeTickerMapper implements Mapper<'ascendex', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: AscendexFuturesData | AscendexTrade) {
return message.m === 'futures-pricing-data' || message.m === 'trades'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'futures-pricing-data',
symbols: [] as string[]
} as const,
{
channel: 'trades',
symbols
} as const
]
}
*map(message: AscendexFuturesData | AscendexTrade, localTimestamp: Date): IterableIterator {
if (message.m === 'trades') {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.symbol, 'ascendex')
pendingTickerInfo.updateLastPrice(Number(message.data[message.data.length - 1].p))
return
}
for (const futuresData of message.con) {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(futuresData.s, 'ascendex')
pendingTickerInfo.updateIndexPrice(Number(futuresData.ip))
pendingTickerInfo.updateMarkPrice(Number(futuresData.mp))
pendingTickerInfo.updateOpenInterest(Number(futuresData.oi))
pendingTickerInfo.updateTimestamp(new Date(futuresData.t))
pendingTickerInfo.updateFundingTimestamp(new Date(futuresData.f))
pendingTickerInfo.updateFundingRate(Number(futuresData.r))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
export class AscendexBookTickerMapper implements Mapper<'ascendex', BookTicker> {
canHandle(message: AscendexTicker) {
return message.m === 'bbo'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'bbo',
symbols
} as const
]
}
*map(message: AscendexTicker, localTimestamp: Date): IterableIterator {
const ask = message.data.ask
const bid = message.data.bid
yield {
type: 'book_ticker',
symbol: message.symbol,
exchange: 'ascendex',
askAmount: ask !== undefined && ask[1] !== undefined ? Number(ask[1]) : undefined,
askPrice: ask !== undefined && ask[0] !== undefined ? Number(ask[0]) : undefined,
bidPrice: bid !== undefined && bid[0] !== undefined ? Number(bid[0]) : undefined,
bidAmount: bid !== undefined && bid[1] !== undefined ? Number(bid[1]) : undefined,
timestamp: new Date(message.data.ts),
localTimestamp: localTimestamp
}
}
}
type AscendexTrade = {
m: 'trades'
symbol: string
data: [{ p: string; q: string; ts: number; bm: boolean; seqnum: number }]
}
type AscendexPriceLevel = [string, string]
type AscendexDepthRealTime = {
m: 'depth-realtime'
symbol: 'XRP/USDT'
data: { ts: 1621814400204; seqnum: 39862426; asks: AscendexPriceLevel[]; bids: AscendexPriceLevel[] }
}
type AscendexDepthRealTimeSnapshot = {
m: 'depth-snapshot-realtime'
symbol: 'XRP/USDT'
data: {
ts: 0
seqnum: 39862426
asks: AscendexPriceLevel[]
bids: AscendexPriceLevel[]
}
}
type AscendexTicker = { m: 'bbo'; symbol: string; data: { ts: number; bid?: AscendexPriceLevel; ask?: AscendexPriceLevel } }
type AscendexFuturesData = {
m: 'futures-pricing-data'
con: [
{
t: 1621814404114
s: 'BTC-PERP'
mp: '34878.075977904'
ip: '34697.17'
oi: '80.6126'
r: '0.000093633'
f: 1621843200000
fi: 28800000
}
]
}
================================================
FILE: src/mappers/binance.ts
================================================
import { debug } from '../debug.ts'
import { CircularBuffer, fromMicroSecondsToDate, lowerCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Exchange, FilterForExchange, Liquidation, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md
export class BinanceTradesMapper
implements Mapper<'binance' | 'binance-jersey' | 'binance-us' | 'binance-futures' | 'binance-delivery', Trade>
{
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.endsWith('@trade')
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
} as const
]
}
*map(binanceTradeResponse: BinanceResponse, localTimestamp: Date) {
const binanceTrade = binanceTradeResponse.data
const isOffBookTrade = binanceTrade.X === 'INSURANCE_FUND' || binanceTrade.X === 'ADL' || binanceTrade.X === 'NA'
if (isOffBookTrade) {
return
}
const trade: Trade = {
type: 'trade',
symbol: binanceTrade.s,
exchange: this._exchange,
id: String(binanceTrade.t),
price: Number(binanceTrade.p),
amount: Number(binanceTrade.q),
side: binanceTrade.m ? 'sell' : 'buy',
timestamp: fromMicroSecondsToDate(binanceTrade.T),
localTimestamp: localTimestamp
}
yield trade
}
}
export class BinanceBookChangeMapper
implements Mapper<'binance' | 'binance-jersey' | 'binance-us' | 'binance-futures' | 'binance-delivery', BookChange>
{
protected readonly symbolToDepthInfoMapping: {
[key: string]: LocalDepthInfo
} = {}
constructor(protected readonly exchange: Exchange, protected readonly ignoreBookSnapshotOverlapError: boolean) {}
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.includes('@depth')
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'depth',
symbols
} as const,
{
channel: 'depthSnapshot',
symbols
} as const
]
}
*map(message: BinanceResponse, localTimestamp: Date) {
const symbol = message.stream.split('@')[0].toUpperCase()
if (this.symbolToDepthInfoMapping[symbol] === undefined) {
this.symbolToDepthInfoMapping[symbol] = {
bufferedUpdates: new CircularBuffer(2000)
}
}
const symbolDepthInfo = this.symbolToDepthInfoMapping[symbol]
const snapshotAlreadyProcessed = symbolDepthInfo.snapshotProcessed
// first check if received message is snapshot and process it as such if it is
if (message.data.lastUpdateId !== undefined) {
// if we've already received 'manual' snapshot, ignore if there is another one
if (snapshotAlreadyProcessed) {
return
}
// produce snapshot book_change
const binanceDepthSnapshotData = message.data
// mark given symbol depth info that has snapshot processed
symbolDepthInfo.lastUpdateId = binanceDepthSnapshotData.lastUpdateId
symbolDepthInfo.snapshotProcessed = true
// if there were any depth updates buffered, let's proccess those by adding to or updating the initial snapshot
for (const update of symbolDepthInfo.bufferedUpdates.items()) {
const bookChange = this.mapBookDepthUpdate(update, localTimestamp)
if (bookChange !== undefined) {
for (const bid of update.b) {
const matchingBid = binanceDepthSnapshotData.bids.find((b) => b[0] === bid[0])
if (matchingBid !== undefined) {
matchingBid[1] = bid[1]
} else {
binanceDepthSnapshotData.bids.push(bid)
}
}
for (const ask of update.a) {
const matchingAsk = binanceDepthSnapshotData.asks.find((a) => a[0] === ask[0])
if (matchingAsk !== undefined) {
matchingAsk[1] = ask[1]
} else {
binanceDepthSnapshotData.asks.push(ask)
}
}
}
}
// remove all buffered updates
symbolDepthInfo.bufferedUpdates.clear()
const bookChange: BookChange = {
type: 'book_change',
symbol,
exchange: this.exchange,
isSnapshot: true,
bids: binanceDepthSnapshotData.bids.map(this.mapBookLevel),
asks: binanceDepthSnapshotData.asks.map(this.mapBookLevel),
timestamp: binanceDepthSnapshotData.T !== undefined ? fromMicroSecondsToDate(binanceDepthSnapshotData.T) : localTimestamp,
localTimestamp
}
yield bookChange
} else if (snapshotAlreadyProcessed) {
// snapshot was already processed let's map the message as normal book_change
const bookChange = this.mapBookDepthUpdate(message.data as BinanceDepthData, localTimestamp)
if (bookChange !== undefined) {
yield bookChange
}
} else {
const binanceDepthUpdateData = message.data as BinanceDepthData
symbolDepthInfo.bufferedUpdates.append(binanceDepthUpdateData)
}
}
protected mapBookDepthUpdate(binanceDepthUpdateData: BinanceDepthData, localTimestamp: Date): BookChange | undefined {
// we can safely assume here that depthContext and lastUpdateId aren't null here as this is method only works
// when we've already processed the snapshot
const depthContext = this.symbolToDepthInfoMapping[binanceDepthUpdateData.s]!
const lastUpdateId = depthContext.lastUpdateId!
// Drop any event where u is <= lastUpdateId in the snapshot
if (binanceDepthUpdateData.u <= lastUpdateId) {
return
}
// The first processed event should have U <= lastUpdateId+1 AND u >= lastUpdateId+1.
if (!depthContext.validatedFirstUpdate) {
// if there is new instrument added it can have empty book at first and that's normal
const bookSnapshotIsEmpty = lastUpdateId == -1
if ((binanceDepthUpdateData.U <= lastUpdateId + 1 && binanceDepthUpdateData.u >= lastUpdateId + 1) || bookSnapshotIsEmpty) {
depthContext.validatedFirstUpdate = true
} else {
const message = `Book depth snaphot has no overlap with first update, update ${JSON.stringify(
binanceDepthUpdateData
)}, lastUpdateId: ${lastUpdateId}, exchange ${this.exchange}`
if (this.ignoreBookSnapshotOverlapError) {
depthContext.validatedFirstUpdate = true
debug(message)
} else {
throw new Error(message)
}
}
}
return {
type: 'book_change',
symbol: binanceDepthUpdateData.s,
exchange: this.exchange,
isSnapshot: false,
bids: binanceDepthUpdateData.b.map(this.mapBookLevel),
asks: binanceDepthUpdateData.a.map(this.mapBookLevel),
timestamp: fromMicroSecondsToDate(binanceDepthUpdateData.E),
localTimestamp: localTimestamp
}
}
protected mapBookLevel(level: BinanceBookLevel) {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
}
export class BinanceFuturesBookChangeMapper
extends BinanceBookChangeMapper
implements Mapper<'binance-futures' | 'binance-delivery', BookChange>
{
constructor(protected readonly exchange: Exchange, protected readonly ignoreBookSnapshotOverlapError: boolean) {
super(exchange, ignoreBookSnapshotOverlapError)
}
protected mapBookDepthUpdate(binanceDepthUpdateData: BinanceFuturesDepthData, localTimestamp: Date): BookChange | undefined {
// we can safely assume here that depthContext and lastUpdateId aren't null here as this is method only works
// when we've already processed the snapshot
const depthContext = this.symbolToDepthInfoMapping[binanceDepthUpdateData.s]!
const lastUpdateId = depthContext.lastUpdateId!
// based on https://binanceapitest.github.io/Binance-Futures-API-doc/wss/#how-to-manage-a-local-order-book-correctly
// Drop any event where u is < lastUpdateId in the snapshot
if (binanceDepthUpdateData.u < lastUpdateId) {
return
}
// The first processed should have U <= lastUpdateId AND u >= lastUpdateId
if (!depthContext.validatedFirstUpdate) {
if (binanceDepthUpdateData.U <= lastUpdateId && binanceDepthUpdateData.u >= lastUpdateId) {
depthContext.validatedFirstUpdate = true
} else {
const message = `Book depth snaphot has no overlap with first update, update ${JSON.stringify(
binanceDepthUpdateData
)}, lastUpdateId: ${lastUpdateId}, exchange ${this.exchange}`
if (this.ignoreBookSnapshotOverlapError) {
depthContext.validatedFirstUpdate = true
debug(message)
} else {
throw new Error(message)
}
}
}
return {
type: 'book_change',
symbol: binanceDepthUpdateData.s,
exchange: this.exchange,
isSnapshot: false,
bids: binanceDepthUpdateData.b.map(this.mapBookLevel),
asks: binanceDepthUpdateData.a.map(this.mapBookLevel),
timestamp: fromMicroSecondsToDate(binanceDepthUpdateData.E),
localTimestamp: localTimestamp
}
}
}
export class BinanceFuturesDerivativeTickerMapper implements Mapper<'binance-futures' | 'binance-delivery', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
private readonly _indexPrices = new Map()
constructor(protected readonly exchange: Exchange) {}
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return (
message.stream.includes('@markPrice') ||
message.stream.endsWith('@ticker') ||
message.stream.endsWith('@openInterest') ||
message.stream.includes('@indexPrice')
)
}
getFilters(symbols?: string[]): FilterForExchange['binance-futures' | 'binance-delivery'][] {
symbols = lowerCaseSymbols(symbols)
const filters = [
{
channel: 'markPrice',
symbols
} as const,
{
channel: 'ticker',
symbols
} as const,
{
channel: 'openInterest',
symbols
} as const
]
if (this.exchange === 'binance-delivery') {
// index channel requires index symbol
filters.push({
channel: 'indexPrice' as any,
symbols: symbols !== undefined ? symbols.map((s) => s.split('_')[0]) : undefined
})
}
return filters
}
*map(
message: BinanceResponse<
BinanceFuturesMarkPriceData | BinanceFuturesTickerData | BinanceFuturesOpenInterestData | BinanceFuturesIndexPriceData
>,
localTimestamp: Date
): IterableIterator {
if (message.data.e === 'indexPriceUpdate') {
this._indexPrices.set(message.data.i, Number(message.data.p))
} else {
const symbol = 's' in message.data ? message.data.s : message.data.symbol
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(symbol, this.exchange)
const lastIndexPrice = this._indexPrices.get(symbol.split('_')[0])
if (lastIndexPrice !== undefined) {
pendingTickerInfo.updateIndexPrice(lastIndexPrice)
}
if (message.data.e === 'markPriceUpdate') {
if ('r' in message.data && message.data.r !== '' && message.data.T !== 0) {
// only perpetual futures have funding rate info in mark price
// delivery futures sometimes send empty ('') r value
pendingTickerInfo.updateFundingRate(Number(message.data.r))
pendingTickerInfo.updateFundingTimestamp(new Date(message.data.T!))
}
if (message.data.i !== undefined) {
pendingTickerInfo.updateIndexPrice(Number(message.data.i))
}
pendingTickerInfo.updateMarkPrice(Number(message.data.p))
pendingTickerInfo.updateTimestamp(fromMicroSecondsToDate(message.data.E))
}
if (message.data.e === '24hrTicker') {
pendingTickerInfo.updateLastPrice(Number(message.data.c))
pendingTickerInfo.updateTimestamp(fromMicroSecondsToDate(message.data.E))
}
if ('openInterest' in message.data) {
pendingTickerInfo.updateOpenInterest(Number(message.data.openInterest))
}
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
export class BinanceLiquidationsMapper implements Mapper<'binance-futures' | 'binance-delivery', Liquidation> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.endsWith('@forceOrder')
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'forceOrder',
symbols
} as const
]
}
*map(binanceTradeResponse: BinanceResponse, localTimestamp: Date) {
const binanceLiquidation = binanceTradeResponse.data.o
// not sure if order status can be different to 'FILLED' for liquidations in practice, but...
if (binanceLiquidation.X !== 'FILLED') {
return
}
const liquidation: Liquidation = {
type: 'liquidation',
symbol: binanceLiquidation.s,
exchange: this._exchange,
id: undefined,
price: Number(binanceLiquidation.p),
amount: Number(binanceLiquidation.z), // Order Filled Accumulated Quantity
side: binanceLiquidation.S === 'SELL' ? 'sell' : 'buy',
timestamp: fromMicroSecondsToDate(binanceLiquidation.T),
localTimestamp: localTimestamp
}
yield liquidation
}
}
export class BinanceBookTickerMapper implements Mapper<'binance-futures' | 'binance-delivery' | 'binance', BookTicker> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.endsWith('@bookTicker')
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'bookTicker',
symbols
} as const
]
}
*map(binanceBookTickerResponse: BinanceResponse, localTimestamp: Date) {
const binanceBookTicker = binanceBookTickerResponse.data
const ticker: BookTicker = {
type: 'book_ticker',
symbol: binanceBookTicker.s,
exchange: this._exchange,
askAmount: binanceBookTicker.A !== undefined ? Number(binanceBookTicker.A) : undefined,
askPrice: binanceBookTicker.a !== undefined ? Number(binanceBookTicker.a) : undefined,
bidPrice: binanceBookTicker.b !== undefined ? Number(binanceBookTicker.b) : undefined,
bidAmount: binanceBookTicker.B !== undefined ? Number(binanceBookTicker.B) : undefined,
timestamp: binanceBookTicker.E !== undefined ? fromMicroSecondsToDate(binanceBookTicker.E) : localTimestamp,
localTimestamp: localTimestamp
}
yield ticker
}
}
type BinanceResponse = {
stream: string
data: T
}
type BinanceTradeData = {
s: string
t: number
p: string
q: string
T: number
m: true
X?: 'INSURANCE_FUND' | 'MARKET' | 'ADL' | 'NA'
}
type BinanceBookLevel = [string, string]
type BinanceDepthData = {
lastUpdateId: undefined
E: number
s: string
U: number
u: number
b: BinanceBookLevel[]
a: BinanceBookLevel[]
}
// T is the time that updated in matching engine, while E is when pushing out from ws server
type BinanceFuturesDepthData = BinanceDepthData & {
pu: number
T: number
}
type BinanceDepthSnapshotData = {
lastUpdateId: number
bids: BinanceBookLevel[]
asks: BinanceBookLevel[]
T?: number
}
type LocalDepthInfo = {
bufferedUpdates: CircularBuffer
snapshotProcessed?: boolean
lastUpdateId?: number
validatedFirstUpdate?: boolean
}
type BinanceFuturesMarkPriceData = {
e: 'markPriceUpdate'
s: string // Symbol
E: number // Event time
p: string // Mark price
r?: string // Funding rate
T?: number // Next funding time
i?: string
}
type BinanceFuturesTickerData = {
e: '24hrTicker'
E: number // Event time
s: string // Symbol
c: string // Last price
}
type BinanceFuturesOpenInterestData = {
e: undefined
symbol: string
openInterest: string
}
type BinanceFuturesIndexPriceData = {
e: 'indexPriceUpdate' // Event type
E: 1591261236000 // Event time
i: string // Pair
p: string // Index Price
}
type BinanceFuturesForceOrderData = {
o: {
s: string // Symbol
S: string // Side
q: string // Original Quantity
p: string // Price
ap: string // Average Price
X: 'FILLED' // Order Status
l: '0.014' // Order Last Filled Quantity
T: 1568014460893 // Order Trade Time
z: string // Order Filled Accumulated Quantity
}
}
type BinanceBookTickerData = {
u: number // order book updateId
s: string // symbol
b: string // best bid price
B: string // best bid qty
a: string // best ask price
A: string // best ask qty
E?: number // transaction time
}
================================================
FILE: src/mappers/binancedex.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
// https://docs.binance.org/api-reference/dex-api/ws-streams.html
export const binanceDexTradesMapper: Mapper<'binance-dex', Trade> = {
canHandle(message: BinanceDexResponse) {
return message.stream === 'trades'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
}
]
},
*map(binanceDexTradeResponse: BinanceDexResponse, localTimestamp: Date): IterableIterator {
for (const binanceDexTrade of binanceDexTradeResponse.data) {
yield {
type: 'trade',
symbol: binanceDexTrade.s,
exchange: 'binance-dex',
id: binanceDexTrade.t,
price: Number(binanceDexTrade.p),
amount: Number(binanceDexTrade.q),
side: binanceDexTrade.tt === 2 ? 'sell' : 'buy',
timestamp: new Date(Math.floor(binanceDexTrade.T / 1000000)),
localTimestamp: localTimestamp
}
}
}
}
const mapBookLevel = (level: BinanceDexBookLevel) => {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
export const binanceDexBookChangeMapper: Mapper<'binance-dex', BookChange> = {
canHandle(message: BinanceDexResponse) {
return message.stream === 'marketDiff' || message.stream === 'depthSnapshot'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'depthSnapshot',
symbols
},
{
channel: 'marketDiff',
symbols
}
]
},
*map(
message: BinanceDexResponse,
localTimestamp: Date
): IterableIterator {
if ('symbol' in message.data) {
// we've got snapshot message
yield {
type: 'book_change',
symbol: message.data.symbol,
exchange: 'binance-dex',
isSnapshot: true,
bids: message.data.bids.map(mapBookLevel),
asks: message.data.asks.map(mapBookLevel),
timestamp: localTimestamp,
localTimestamp
}
} else {
// we've got update
yield {
type: 'book_change',
symbol: message.data.s,
exchange: 'binance-dex',
isSnapshot: false,
bids: message.data.b.map(mapBookLevel),
asks: message.data.a.map(mapBookLevel),
timestamp: localTimestamp,
localTimestamp
}
}
}
}
export const binanceDexBookTickerMapper: Mapper<'binance-dex', BookTicker> = {
canHandle(message: BinanceDexResponse) {
return message.stream === 'ticker'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'ticker',
symbols
}
]
},
*map(binanceDexTradeResponse: BinanceDexResponse, localTimestamp: Date): IterableIterator {
const binanceDexTicker = binanceDexTradeResponse.data
const ticker: BookTicker = {
type: 'book_ticker',
symbol: binanceDexTicker.s,
exchange: 'binance-dex',
askAmount: binanceDexTicker.A !== undefined ? Number(binanceDexTicker.A) : undefined,
askPrice: binanceDexTicker.a !== undefined ? Number(binanceDexTicker.a) : undefined,
bidPrice: binanceDexTicker.b !== undefined ? Number(binanceDexTicker.b) : undefined,
bidAmount: binanceDexTicker.B !== undefined ? Number(binanceDexTicker.B) : undefined,
timestamp: binanceDexTicker.E !== undefined ? new Date(binanceDexTicker.E * 1000) : localTimestamp,
localTimestamp: localTimestamp
}
yield ticker
}
}
type BinanceDexResponse = {
stream: string
data: T
}
type BinanceDexTradeData = {
s: string // Symbol
t: string // Trade ID
p: string // Price
q: string // Quantity
T: number // Trade time
tt: number //tiekertype 0: Unknown 1: SellTaker 2: BuyTaker 3: BuySurplus 4: SellSurplus 5: Neutral
}[]
type BinanceDexBookLevel = [string, string]
type BinanceDexDepthSnapshotData = {
symbol: string
bids: BinanceDexBookLevel[]
asks: BinanceDexBookLevel[]
}
type BinanceDexMarketDiffData = {
E: number // Event time
s: string // Symbol
b: BinanceDexBookLevel[]
a: BinanceDexBookLevel[]
}
type BinanceDexTickerData = {
e: '24hrTicker' // Event type
E: 123456789 // Event time
s: 'ABC_0DX-BNB' // Symbol
b: '0.0024' // Best bid price
B: '10' // Best bid quantity
a: '0.0026' // Best ask price
A: '100' // Best ask quantity
}
================================================
FILE: src/mappers/binanceeuropeanoptions.ts
================================================
import { asNumberIfValid, lowerCaseSymbols, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, OptionSummary, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
// https://binance-docs.github.io/apidocs/voptions/en/#websocket-market-streams
export class BinanceEuropeanOptionsTradesMapper implements Mapper<'binance-european-options', Trade> {
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.endsWith('@trade')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
} as const
]
}
*map(binanceTradeResponse: BinanceResponse, localTimestamp: Date) {
const trade: Trade = {
type: 'trade',
symbol: binanceTradeResponse.data.s,
exchange: 'binance-european-options',
id: binanceTradeResponse.data.t,
price: Number(binanceTradeResponse.data.p),
amount: Number(binanceTradeResponse.data.q),
side: binanceTradeResponse.data.S === '-1' ? 'sell' : 'buy',
timestamp: new Date(binanceTradeResponse.data.T),
localTimestamp: localTimestamp
}
yield trade
}
}
export class BinanceEuropeanOptionsTradesMapperV2 implements Mapper<'binance-european-options', Trade> {
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.endsWith('@optionTrade')
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'optionTrade',
symbols
} as const
]
}
*map(binanceTradeResponse: BinanceResponse, localTimestamp: Date) {
const trade: Trade = {
type: 'trade',
symbol: binanceTradeResponse.data.s,
exchange: 'binance-european-options',
id: String(binanceTradeResponse.data.t),
price: Number(binanceTradeResponse.data.p),
amount: Number(binanceTradeResponse.data.q),
side: binanceTradeResponse.data.m ? 'sell' : 'buy',
timestamp: new Date(binanceTradeResponse.data.T),
localTimestamp: localTimestamp
}
yield trade
}
}
export class BinanceEuropeanOptionsBookChangeMapper implements Mapper<'binance-european-options', BookChange> {
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.includes('@depth100')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'depth100',
symbols
} as const
]
}
*map(message: BinanceResponse, localTimestamp: Date) {
const bookChange: BookChange = {
type: 'book_change',
symbol: message.data.s,
exchange: 'binance-european-options',
isSnapshot: true,
bids: message.data.b.map(this.mapBookLevel),
asks: message.data.a.map(this.mapBookLevel),
timestamp: message.data.E !== undefined ? new Date(message.data.E) : new Date(message.data.T),
localTimestamp
}
yield bookChange
}
protected mapBookLevel(level: BinanceBookLevel) {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
}
export class BinanceEuropeanOptionsBookChangeMapperV2 implements Mapper<'binance-european-options', BookChange> {
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.includes('@depth20')
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'depth20',
symbols
} as const
]
}
*map(message: BinanceResponse, localTimestamp: Date) {
const bookChange: BookChange = {
type: 'book_change',
symbol: message.data.s,
exchange: 'binance-european-options',
isSnapshot: true,
bids: message.data.b.map(this.mapBookLevel),
asks: message.data.a.map(this.mapBookLevel),
timestamp: new Date(message.data.T),
localTimestamp
}
yield bookChange
}
protected mapBookLevel(level: BinanceBookLevel) {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
}
export class BinanceEuropeanOptionsBookTickerMapper implements Mapper<'binance-european-options', BookTicker> {
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.endsWith('@bookTicker')
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'bookTicker',
symbols
} as const
]
}
*map(message: BinanceResponse, localTimestamp: Date) {
const bestBidPrice = Number(message.data.b)
const bestBidAmount = Number(message.data.B)
const bestAskPrice = Number(message.data.a)
const bestAskAmount = Number(message.data.A)
const bookTicker: BookTicker = {
type: 'book_ticker',
symbol: message.data.s,
exchange: 'binance-european-options',
bidPrice: bestBidPrice > 0 ? bestBidPrice : undefined,
bidAmount: bestBidAmount > 0 ? bestBidAmount : undefined,
askPrice: bestAskPrice > 0 ? bestAskPrice : undefined,
askAmount: bestAskAmount > 0 ? bestAskAmount : undefined,
timestamp: new Date(message.data.T),
localTimestamp
}
yield bookTicker
}
}
export class BinanceEuropeanOptionSummaryMapper implements Mapper<'binance-european-options', OptionSummary> {
private readonly _indexPrices = new Map()
private readonly _openInterests = new Map()
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return message.stream.endsWith('@ticker') || message.stream.endsWith('@index') || message.stream.includes('@openInterest')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
const indexes =
symbols !== undefined
? symbols.map((s) => {
const symbolParts = s.split('-')
return `${symbolParts[0]}USDT`
})
: undefined
const underlyings =
symbols !== undefined
? symbols.map((s) => {
const symbolParts = s.split('-')
return `${symbolParts[0]}`
})
: undefined
return [
{
channel: 'ticker',
symbols
} as const,
{
channel: 'index',
symbols: indexes
} as const,
{
channel: 'openInterest',
symbols: underlyings
} as const
]
}
*map(
message: BinanceResponse,
localTimestamp: Date
) {
if (message.stream.endsWith('@index')) {
const lastIndexPrice = Number((message.data as any).p)
if (lastIndexPrice > 0) {
this._indexPrices.set((message.data as any).s, lastIndexPrice)
}
return
}
if (message.stream.includes('@openInterest')) {
for (let data of message.data as BinanceOptionsOpenInterestData[]) {
const openInterest = Number(data.o)
if (openInterest >= 0) {
this._openInterests.set(data.s, openInterest)
}
}
return
}
const optionInfo = message.data as BinanceOptionsTickerData
const [base, expiryPart, strikePrice, optionType] = optionInfo.s.split('-')
const expirationDate = new Date(`20${expiryPart.slice(0, 2)}-${expiryPart.slice(2, 4)}-${expiryPart.slice(4, 6)}Z`)
expirationDate.setUTCHours(8)
const isPut = optionType === 'P'
const underlyingIndex = `${base}USDT`
let bestBidPrice = asNumberIfValid(optionInfo.bo)
if (bestBidPrice === 0) {
bestBidPrice = undefined
}
let bestBidAmount = asNumberIfValid(optionInfo.bq)
if (bestBidAmount === 0) {
bestBidAmount = undefined
}
let bestAskPrice = asNumberIfValid(optionInfo.ao)
if (bestAskPrice === 0) {
bestAskPrice = undefined
}
let bestAskAmount = asNumberIfValid(optionInfo.aq)
if (bestAskAmount === 0) {
bestAskAmount = undefined
}
let bestBidIV = bestBidPrice !== undefined ? asNumberIfValid(optionInfo.b) : undefined
if (bestBidIV === -1) {
bestBidIV = undefined
}
let bestAskIV = bestAskPrice !== undefined ? asNumberIfValid(optionInfo.a) : undefined
if (bestAskIV === -1) {
bestAskIV = undefined
}
const optionSummary: OptionSummary = {
type: 'option_summary',
symbol: optionInfo.s,
exchange: 'binance-european-options',
optionType: isPut ? 'put' : 'call',
strikePrice: Number(strikePrice),
expirationDate,
bestBidPrice,
bestBidAmount,
bestBidIV,
bestAskPrice,
bestAskAmount,
bestAskIV,
lastPrice: asNumberIfValid(optionInfo.c),
openInterest: this._openInterests.get(optionInfo.s),
markPrice: asNumberIfValid(optionInfo.mp),
markIV: undefined,
delta: asNumberIfValid(optionInfo.d),
gamma: asNumberIfValid(optionInfo.g),
vega: asNumberIfValid(optionInfo.v),
theta: asNumberIfValid(optionInfo.t),
rho: undefined,
underlyingPrice: this._indexPrices.get(underlyingIndex),
underlyingIndex,
timestamp: new Date(optionInfo.E),
localTimestamp: localTimestamp
}
yield optionSummary
}
}
export class BinanceEuropeanOptionSummaryMapperV2 implements Mapper<'binance-european-options', OptionSummary> {
private readonly _lastPrices = new Map()
private readonly _openInterests = new Map()
canHandle(message: BinanceResponse) {
if (message.stream === undefined) {
return false
}
return (
message.stream.endsWith('@optionMarkPrice') ||
message.stream.endsWith('@optionTicker') ||
message.stream.includes('@optionOpenInterest')
)
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
const underlyings =
symbols !== undefined
? symbols.map((s) => {
const symbolParts = s.split('-')
return `${symbolParts[0]}usdt`
})
: undefined
return [
{
channel: 'optionMarkPrice',
symbols: underlyings
} as const,
{
channel: 'optionTicker',
symbols
} as const,
{
channel: 'optionOpenInterest',
symbols: underlyings
} as const
]
}
*map(
message: BinanceResponse,
localTimestamp: Date
) {
// Handle optionTicker messages to track last prices
if (message.stream.endsWith('@optionTicker')) {
const tickerData = message.data as BinanceOptionsTickerData
const lastPrice = Number(tickerData.c)
if (lastPrice > 0) {
this._lastPrices.set(tickerData.s, lastPrice)
}
return
}
// Handle optionOpenInterest messages to track open interest
if (message.stream.includes('@optionOpenInterest')) {
const openInterestArray = message.data as BinanceOptionsOpenInterestDataV2[]
for (let oi of openInterestArray) {
const openInterest = Number(oi.o)
if (openInterest >= 0) {
this._openInterests.set(oi.s, openInterest)
}
}
return
}
// optionMarkPrice contains all data needed: greeks, IV, best bid/ask, mark price, and index price
const markPriceArray = message.data as BinanceOptionsMarkPriceData[]
for (let markData of markPriceArray) {
const [base, expiryPart, strikePrice, optionType] = markData.s.split('-')
const expirationDate = new Date(`20${expiryPart.slice(0, 2)}-${expiryPart.slice(2, 4)}-${expiryPart.slice(4, 6)}Z`)
expirationDate.setUTCHours(8)
const isPut = optionType === 'P'
const underlyingIndex = `${base}USDT`
let bestBidPrice = asNumberIfValid(markData.bo)
if (bestBidPrice === 0) {
bestBidPrice = undefined
}
let bestBidAmount = asNumberIfValid(markData.bq)
if (bestBidAmount === 0) {
bestBidAmount = undefined
}
let bestAskPrice = asNumberIfValid(markData.ao)
if (bestAskPrice === 0) {
bestAskPrice = undefined
}
let bestAskAmount = asNumberIfValid(markData.aq)
if (bestAskAmount === 0) {
bestAskAmount = undefined
}
let bestBidIV = bestBidPrice !== undefined ? asNumberIfValid(markData.b) : undefined
if (bestBidIV === -1) {
bestBidIV = undefined
}
let bestAskIV = bestAskPrice !== undefined ? asNumberIfValid(markData.a) : undefined
if (bestAskIV === -1) {
bestAskIV = undefined
}
const markPrice = asNumberIfValid(markData.mp)
const markIV = asNumberIfValid(markData.vo)
const delta = asNumberIfValid(markData.d)
const gamma = asNumberIfValid(markData.g)
const vega = asNumberIfValid(markData.v)
const theta = asNumberIfValid(markData.t)
const underlyingPrice = asNumberIfValid(markData.i) // Index price is included in mark price data
const optionSummary: OptionSummary = {
type: 'option_summary',
symbol: markData.s,
exchange: 'binance-european-options',
optionType: isPut ? 'put' : 'call',
strikePrice: Number(strikePrice),
expirationDate,
bestBidPrice,
bestBidAmount,
bestBidIV,
bestAskPrice,
bestAskAmount,
bestAskIV,
lastPrice: this._lastPrices.get(markData.s),
openInterest: this._openInterests.get(markData.s),
markPrice,
markIV,
delta,
gamma,
vega,
theta,
rho: undefined,
underlyingPrice,
underlyingIndex,
timestamp: new Date(markData.E),
localTimestamp: localTimestamp
}
yield optionSummary
}
}
}
type BinanceResponse = {
stream: string
data: T
}
type BinanceOptionsTradeData = {
e: 'trade'
E: 1696118408137
s: 'DOGE-231006-0.06-C'
t: '15'
p: '2.64'
q: '0.01'
b: '4647850284614262784'
a: '4719907951072796672'
T: 1696118408134
S: '-1'
}
type BinanceOptionsDepthData = {
e: 'depth'
E: 1696118400038
T: 1696118399082
s: 'BTC-231027-34000-C'
u: 1925729
pu: 1925729
b: [['60', '7.31'], ['55', '2.5'], ['50', '15'], ['45', '15'], ['40', '34.04']]
a: [['65', '8.28'], ['70', '38.88'], ['75', '15'], ['1200', '0.01'], ['4660', '0.42']]
}
type BinanceOptionsTickerData = {
e: '24hrTicker'
E: 1696118400043
T: 1696118400000
s: 'BNB-231013-200-P'
o: '1'
h: '1'
l: '0.9'
c: '0.9'
V: '11.08'
A: '9.97'
P: '-0.1'
p: '-0.1'
Q: '11'
F: '0'
L: '8'
n: 1
bo: '1'
ao: '1.7'
bq: '50'
aq: '50'
b: '0.35929501'
a: '0.43317497'
d: '-0.16872899'
t: '-0.16779034'
g: '0.0153237'
v: '0.09935076'
vo: '0.41658748'
mp: '1.5'
hl: '37.1'
ll: '0.1'
eep: '0'
}
type BinanceOptionsIndexData = { e: 'index'; E: 1696118400040; s: 'BNBUSDT'; p: '214.6133998' }
type BinanceOptionsOpenInterestData = { e: 'openInterest'; E: 1696118400042; s: 'XRP-231006-0.46-P'; o: '39480.0'; h: '20326.64319' }
type BinanceBookLevel = [string, string]
// V2 Types for new format (Dec 17, 2025+)
type BinanceOptionsTradeDataV2 = {
e: 'trade'
E: number // event time
T: number // trade completed time
s: string // option symbol
t: number // trade ID
p: string // price
q: string // quantity
X: 'MARKET' | 'BLOCK' // trade type
S: 'BUY' | 'SELL' // direction
m: boolean // is buyer market maker
}
type BinanceOptionsDepthDataV2 = {
e: 'depthUpdate'
E: number // event time
T: number // transaction time
s: string // symbol
U: number // first update ID
u: number // final update ID
pu: number // previous final update ID
b: [string, string][] // bids
a: [string, string][] // asks
}
type BinanceOptionsBookTickerData = {
e: 'bookTicker'
u: number // order book update ID
s: string // symbol
b: string // best bid price
B: string // best bid qty
a: string // best ask price
A: string // best ask qty
T: number // transaction time
E: number // event time
}
type BinanceOptionsOpenInterestDataV2 = {
e: 'openInterest'
E: number // event time
s: string // symbol
o: string // open interest (quantity)
h: string // open interest in notional value (USD)
}
type BinanceOptionsMarkPriceData = {
s: string // option symbol
mp: string // mark price
E: number // event time
e: 'markPrice'
i: string // index price
P: string // premium
bo: string // best bid price
ao: string // best ask price
bq: string // best bid quantity
aq: string // best ask quantity
b: string // bid IV
a: string // ask IV
hl: string // high limit price
ll: string // low limit price
vo: string // mark IV
rf: string // risk free rate
d: string // delta
t: string // theta
g: string // gamma
v: string // vega
}
================================================
FILE: src/mappers/bitfinex.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Exchange, FilterForExchange, Liquidation, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://docs.bitfinex.com/v2/docs/ws-general
export class BitfinexTradesMapper implements Mapper<'bitfinex' | 'bitfinex-derivatives', Trade> {
private readonly _channelIdToSymbolMap: Map = new Map()
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BitfinexMessage) {
// non sub messages are provided as arrays
if (Array.isArray(message)) {
// first test if message itself provides channel name and if so if it's trades
const channelName = message[message.length - 2]
if (typeof channelName === 'string') {
return channelName === 'trades'
}
// otherwise use channel to id mapping
return this._channelIdToSymbolMap.get(message[0]) !== undefined
}
// store mapping between channel id and symbols
if (message.event === 'subscribed') {
const isTradeChannel = message.channel === 'trades'
if (isTradeChannel) {
this._channelIdToSymbolMap.set(message.chanId, message.pair)
}
}
return false
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
} as const
]
}
*map(message: BitfinexTrades, localTimestamp: Date) {
const symbolFromMessage = message[message.length - 1]
const symbol = typeof symbolFromMessage === 'string' ? symbolFromMessage : this._channelIdToSymbolMap.get(message[0])
// ignore if we don't have matching symbol
if (symbol === undefined) {
return
}
// ignore heartbeats
if (message[1] === 'hb') {
return
}
// ignore snapshots
if (message[1] !== 'te') {
return
}
const [id, timestamp, amount, price] = message[2]
const trade: Trade = {
type: 'trade',
symbol,
exchange: this._exchange,
id: String(id),
price,
amount: Math.abs(amount),
side: amount < 0 ? 'sell' : 'buy',
timestamp: new Date(timestamp),
localTimestamp: localTimestamp
}
yield trade
}
}
export class BitfinexBookChangeMapper implements Mapper<'bitfinex' | 'bitfinex-derivatives', BookChange> {
private readonly _channelIdToSymbolMap: Map = new Map()
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BitfinexMessage) {
// non sub messages are provided as arrays
if (Array.isArray(message)) {
// first test if message itself provides channel name and if so if it's a book
const channelName = message[message.length - 2]
if (typeof channelName === 'string') {
return channelName === 'book'
}
// otherwise use channel to id mapping
return this._channelIdToSymbolMap.get(message[0]) !== undefined
}
// store mapping between channel id and symbols
if (message.event === 'subscribed') {
const isBookP0Channel = message.channel === 'book' && message.prec === 'P0'
if (isBookP0Channel) {
this._channelIdToSymbolMap.set(message.chanId, message.pair)
}
}
return false
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'book',
symbols
} as const
]
}
*map(message: BitfinexBooks, localTimestamp: Date) {
const symbolFromMessage = message[message.length - 1]
const symbol = typeof symbolFromMessage === 'string' ? symbolFromMessage : this._channelIdToSymbolMap.get(message[0])
// ignore if we don't have matching symbol
if (symbol === undefined) {
return
}
// ignore heartbeats
if (message[1] === 'hb') {
return
}
const isSnapshot = Array.isArray(message[1][0])
const bookLevels = (isSnapshot ? message[1] : [message[1]]) as BitfinexBookLevel[]
const asks = bookLevels.filter((level) => level[2] < 0)
const bids = bookLevels.filter((level) => level[2] > 0)
const bookChange: BookChange = {
type: 'book_change',
symbol,
exchange: this._exchange,
isSnapshot,
bids: bids.map(this._mapBookLevel),
asks: asks.map(this._mapBookLevel),
timestamp: new Date(message[3]),
localTimestamp: localTimestamp
}
yield bookChange
}
private _mapBookLevel(level: BitfinexBookLevel) {
const [price, count, bitfinexAmount] = level
const amount = count === 0 ? 0 : Math.abs(bitfinexAmount)
return { price, amount }
}
}
export class BitfinexDerivativeTickerMapper implements Mapper<'bitfinex-derivatives', DerivativeTicker> {
private readonly _channelIdToSymbolMap: Map = new Map()
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: BitfinexMessage) {
// non sub messages are provided as arrays
if (Array.isArray(message)) {
// first test if message itself provides channel name and if so if it's a status
const channelName = message[message.length - 2]
if (typeof channelName === 'string') {
return channelName === 'status'
}
// otherwise use channel to id mapping
return this._channelIdToSymbolMap.get(message[0]) !== undefined
}
// store mapping between channel id and symbols
if (message.event === 'subscribed') {
const isDerivStatusChannel = message.channel === 'status' && message.key && message.key.startsWith('deriv:')
if (isDerivStatusChannel) {
this._channelIdToSymbolMap.set(message.chanId, message.key!.replace('deriv:t', ''))
}
}
return false
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'status',
symbols
} as const
]
}
*map(message: BitfinexStatusMessage, localTimestamp: Date): IterableIterator {
const symbolFromMessage = message[message.length - 1]
const symbol = typeof symbolFromMessage === 'string' ? symbolFromMessage : this._channelIdToSymbolMap.get(message[0])
// ignore if we don't have matching symbol
if (symbol === undefined) {
return
}
// ignore heartbeats
if (message[1] === 'hb') {
return
}
const statusInfo = message[1]
// https://docs.bitfinex.com/v2/reference#ws-public-status
const fundingRate = statusInfo[11]
const indexPrice = statusInfo[3]
const lastPrice = statusInfo[2]
const markPrice = statusInfo[14]
const openInterest = statusInfo[17]
const nextFundingTimestamp = statusInfo[7]
const predictedFundingRate = statusInfo[8]
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(symbol, 'bitfinex-derivatives')
pendingTickerInfo.updateFundingRate(fundingRate)
pendingTickerInfo.updateFundingTimestamp(nextFundingTimestamp !== undefined ? new Date(nextFundingTimestamp) : undefined)
pendingTickerInfo.updatePredictedFundingRate(predictedFundingRate)
pendingTickerInfo.updateIndexPrice(indexPrice)
pendingTickerInfo.updateLastPrice(lastPrice)
pendingTickerInfo.updateMarkPrice(markPrice)
pendingTickerInfo.updateOpenInterest(openInterest)
pendingTickerInfo.updateTimestamp(new Date(message[3]))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
export class BitfinexLiquidationsMapper implements Mapper<'bitfinex-derivatives', Liquidation> {
private _liquidationsChannelId: number | undefined = undefined
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BitfinexMessage) {
// non sub messages are provided as arrays
if (Array.isArray(message)) {
// first test if message itself provides channel name and if so if it's liquidations
const channelName = message[message.length - 2]
if (typeof channelName === 'string') {
return channelName === 'liquidations'
}
// otherwise use channel id
return this._liquidationsChannelId === message[0]
}
// store liquidation channel id
if (message.event === 'subscribed') {
const isLiquidationsChannel = message.channel === 'status' && message.key === 'liq:global'
if (isLiquidationsChannel) {
this._liquidationsChannelId = message.chanId
}
}
return false
}
getFilters() {
// liquidations channel is global, not per symbol
return [
{
channel: 'liquidations'
} as const
]
}
*map(message: BitfinexLiquidation, localTimestamp: Date) {
// ignore heartbeats
if (message[1] === 'hb') {
return
}
if (!message[1]) {
return
}
// see https://docs.bitfinex.com/reference#ws-public-status
for (let bitfinexLiquidation of message[1]) {
const isInitialLiquidationTrigger = bitfinexLiquidation[8] === 0
// process only initial liquidation triggers not subsequent 'matches', assumption here is that
// there's only single initial liquidation trigger but there can be multiple matches for single liquidation
if (isInitialLiquidationTrigger) {
const id = String(bitfinexLiquidation[1])
const timestamp = new Date(bitfinexLiquidation[2])
const symbol = bitfinexLiquidation[4].replace('t', '')
const price = bitfinexLiquidation[6]
const amount = bitfinexLiquidation[5]
const liquidation: Liquidation = {
type: 'liquidation',
symbol,
exchange: this._exchange,
id,
price,
amount: Math.abs(amount),
side: amount < 0 ? 'buy' : 'sell',
timestamp,
localTimestamp: localTimestamp
}
yield liquidation
}
}
}
}
export class BitfinexBookTickerMapper implements Mapper<'bitfinex' | 'bitfinex-derivatives', BookTicker> {
private readonly _channelIdToSymbolMap: Map = new Map()
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BitfinexMessage) {
// non sub messages are provided as arrays
if (Array.isArray(message)) {
// first test if message itself provides channel name and if so if it's trades
const channelName = message[message.length - 2]
if (typeof channelName === 'string') {
return channelName === 'ticker'
}
// otherwise use channel to id mapping
return this._channelIdToSymbolMap.get(message[0]) !== undefined
}
// store mapping between channel id and symbols
if (message.event === 'subscribed') {
const isTicker = message.channel === 'ticker' && message.pair !== undefined
if (isTicker) {
this._channelIdToSymbolMap.set(message.chanId, message.pair)
}
}
return false
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'ticker',
symbols
} as const
]
}
*map(message: BitfinexTicker, localTimestamp: Date) {
const symbolFromMessage = message[message.length - 1]
const symbol = typeof symbolFromMessage === 'string' ? symbolFromMessage : this._channelIdToSymbolMap.get(message[0])
if (symbol === undefined) {
return
}
// ignore heartbeats
if (message[1] === 'hb') {
return
}
// ignore funding tickers
const tickerData = message[1]
if (tickerData.length > 11) {
return
}
const [bidPrice, bidAmount, askPrice, askAmount] = tickerData
const ticker: BookTicker = {
type: 'book_ticker',
symbol,
exchange: this._exchange,
askAmount,
askPrice,
bidPrice,
bidAmount,
timestamp: new Date(message[3]),
localTimestamp: localTimestamp
}
yield ticker
}
}
type BitfinexMessage =
| {
event: 'subscribed'
channel: FilterForExchange['bitfinex-derivatives']['channel']
chanId: number
pair: string
prec: string
key?: string
}
| Array
type BitfinexHeartbeat = [number, 'hb']
type BitfinexTrades = [number, 'te' | any[], [number, number, number, number]] | BitfinexHeartbeat
type BitfinexBookLevel = [number, number, number]
type BitfinexBooks = [number, BitfinexBookLevel | BitfinexBookLevel[], number, number] | BitfinexHeartbeat
type BitfinexStatusMessage = [number, (number | undefined)[], number, number] | BitfinexHeartbeat
type BitfinexLiquidation =
| [number, ['pos', number, number, null, string, number, number, null, number, number, null, number][]]
| BitfinexHeartbeat
type BitfinexTicker =
| [
CHANNEL_ID: number,
ITEMS:
| [
BID: number,
BID_SIZE: number,
ASK: number,
ASK_SIZE: number,
DAILY_CHANGE: number,
DAILY_CHANGE_RELATIVE: number,
LAST_PRICE: number,
VOLUME: number,
HIGH: number,
LOW: number
]
| [
BID: number,
BID_SIZE: number,
ASK: number,
ASK_SIZE: number,
DAILY_CHANGE: number,
DAILY_CHANGE_RELATIVE: number,
LAST_PRICE: number,
VOLUME: number,
HIGH: number,
LOW: number,
EXTRA: null
],
SEQ_ID: number,
TIMESTAMP: number
]
| BitfinexHeartbeat
================================================
FILE: src/mappers/bitflyer.ts
================================================
import { parseμs, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
export const bitflyerTradesMapper: Mapper<'bitflyer', Trade> = {
canHandle(message: BitflyerExecutions | BitflyerBoard) {
return message.params.channel.startsWith('lightning_executions')
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'lightning_executions',
symbols
}
]
},
*map(bitflyerExecutions: BitflyerExecutions, localTimestamp: Date) {
const symbol = bitflyerExecutions.params.channel.replace('lightning_executions_', '')
for (const execution of bitflyerExecutions.params.message) {
const timestamp = new Date(execution.exec_date)
timestamp.μs = parseμs(execution.exec_date)
const trade: Trade = {
type: 'trade',
symbol,
exchange: 'bitflyer',
id: String(execution.id),
price: execution.price,
amount: execution.size,
side: execution.side === 'BUY' ? 'buy' : execution.side === 'SELL' ? 'sell' : 'unknown',
timestamp,
localTimestamp: localTimestamp
}
yield trade
}
}
}
const mapBookLevel = ({ price, size }: BitflyerBookLevel) => {
return { price, amount: size }
}
export class BitflyerBookChangeMapper implements Mapper<'bitflyer', BookChange> {
private readonly _snapshotsInfo: Map = new Map()
canHandle(message: BitflyerExecutions | BitflyerBoard) {
return message.params.channel.startsWith('lightning_board')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'lightning_board_snapshot',
symbols
} as const,
{
channel: 'lightning_board',
symbols
} as const
]
}
*map(bitflyerBoard: BitflyerBoard, localTimestamp: Date): IterableIterator {
const channel = bitflyerBoard.params.channel
const isSnapshot = channel.startsWith('lightning_board_snapshot_')
const symbol = isSnapshot ? channel.replace('lightning_board_snapshot_', '') : channel.replace('lightning_board_', '')
if (this._snapshotsInfo.has(symbol) === false) {
if (isSnapshot) {
this._snapshotsInfo.set(symbol, true)
} else {
// skip change messages until we've received book snapshot
return
}
}
yield {
type: 'book_change',
symbol,
exchange: 'bitflyer',
isSnapshot,
bids: bitflyerBoard.params.message.bids.map(mapBookLevel),
asks: bitflyerBoard.params.message.asks.map(mapBookLevel),
timestamp: localTimestamp,
localTimestamp
}
}
}
export const bitflyerBookTickerMapper: Mapper<'bitflyer', BookTicker> = {
canHandle(message: BitflyerTicker) {
return message.params.channel.startsWith('lightning_ticker')
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'lightning_ticker',
symbols
}
]
},
*map(bitflyerTickerMessage: BitflyerTicker, localTimestamp: Date) {
const symbol = bitflyerTickerMessage.params.channel.replace('lightning_ticker_', '')
const bitflyerTicker = bitflyerTickerMessage.params.message
const timestamp = new Date(bitflyerTicker.timestamp)
timestamp.μs = parseμs(bitflyerTicker.timestamp)
const ticker: BookTicker = {
type: 'book_ticker',
symbol,
exchange: 'bitflyer',
askAmount: bitflyerTicker.best_ask_size,
askPrice: bitflyerTicker.best_ask,
bidPrice: bitflyerTicker.best_bid,
bidAmount: bitflyerTicker.best_bid_size,
timestamp,
localTimestamp: localTimestamp
}
yield ticker
}
}
type BitflyerExecutions = {
method: 'channelMessage'
params: {
channel: string
message: {
id: number
side: 'SELL' | 'BUY'
price: number
size: number
exec_date: string
}[]
}
}
type BitflyerBookLevel = {
price: number
size: number
}
type BitflyerBoard = {
method: 'channelMessage'
params: {
channel: string
message: {
bids: BitflyerBookLevel[]
asks: BitflyerBookLevel[]
}
}
}
type BitflyerTicker = {
method: 'channelMessage'
params: {
channel: 'lightning_ticker_ETH_JPY'
message: {
product_code: 'ETH_JPY'
state: 'RUNNING'
timestamp: '2021-09-01T00:00:00.2115808Z'
tick_id: 2830807
best_bid: 376592.0
best_ask: 376676.0
best_bid_size: 0.01
best_ask_size: 0.4
total_bid_depth: 5234.4333389
total_ask_depth: 1511.52678
market_bid_size: 0.0
market_ask_size: 0.0
ltp: 376789.0
volume: 37853.5120461
volume_by_product: 37853.5120461
}
}
}
================================================
FILE: src/mappers/bitget.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Exchange, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
export class BitgetTradesMapper implements Mapper<'bitget' | 'bitget-futures', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BitgetTradeMessage) {
return message.arg.channel === 'trade' && message.action === 'update'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
} as const
]
}
*map(message: BitgetTradeMessage, localTimestamp: Date): IterableIterator {
for (let trade of message.data) {
yield {
type: 'trade',
symbol: message.arg.instId,
exchange: this._exchange,
id: trade.tradeId,
price: Number(trade.price),
amount: Number(trade.size),
side: trade.side === 'buy' ? 'buy' : 'sell',
timestamp: new Date(Number(trade.ts)),
localTimestamp: localTimestamp
}
}
}
}
function mapPriceLevel(level: [string, string]) {
return {
price: Number(level[0]),
amount: Number(level[1])
}
}
export class BitgetBookChangeMapper implements Mapper<'bitget' | 'bitget-futures', BookChange> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BitgetOrderbookMessage) {
return message.arg.channel === 'books15' && message.action === 'snapshot'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'books15',
symbols
} as const
]
}
*map(message: BitgetOrderbookMessage, localTimestamp: Date): IterableIterator {
for (let orderbookData of message.data) {
yield {
type: 'book_change',
symbol: message.arg.instId,
exchange: this._exchange,
isSnapshot: message.action === 'snapshot',
bids: orderbookData.bids.map(mapPriceLevel),
asks: orderbookData.asks.map(mapPriceLevel),
timestamp: new Date(Number(orderbookData.ts)),
localTimestamp
}
}
}
}
export class BitgetBookTickerMapper implements Mapper<'bitget' | 'bitget-futures', BookTicker> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BitgetBBoMessage) {
return message.arg.channel === 'books1' && message.action === 'snapshot'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: `books1` as const,
symbols
}
]
}
*map(message: BitgetBBoMessage, localTimestamp: Date): IterableIterator {
for (const bboMessage of message.data) {
const ticker: BookTicker = {
type: 'book_ticker',
symbol: message.arg.instId,
exchange: this._exchange,
askAmount: bboMessage.asks[0] ? Number(bboMessage.asks[0][1]) : undefined,
askPrice: bboMessage.asks[0] ? Number(bboMessage.asks[0][0]) : undefined,
bidPrice: bboMessage.bids[0] ? Number(bboMessage.bids[0][0]) : undefined,
bidAmount: bboMessage.bids[0] ? Number(bboMessage.bids[0][1]) : undefined,
timestamp: new Date(Number(bboMessage.ts)),
localTimestamp: localTimestamp
}
yield ticker
}
}
}
export class BitgetDerivativeTickerMapper implements Mapper<'bitget-futures', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: BitgetTickerMessage) {
return message.arg.channel === 'ticker' && message.action === 'snapshot'
}
getFilters(symbols?: string[]) {
return [
{
channel: 'ticker',
symbols
} as const
]
}
*map(message: BitgetTickerMessage, localTimestamp: Date): IterableIterator {
for (const tickerMessage of message.data) {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(tickerMessage.symbol, 'bitget-futures')
pendingTickerInfo.updateIndexPrice(Number(tickerMessage.indexPrice))
pendingTickerInfo.updateMarkPrice(Number(tickerMessage.markPrice))
pendingTickerInfo.updateOpenInterest(Number(tickerMessage.holdingAmount))
pendingTickerInfo.updateLastPrice(Number(tickerMessage.lastPr))
pendingTickerInfo.updateTimestamp(new Date(Number(tickerMessage.ts)))
if (tickerMessage.nextFundingTime !== '0') {
pendingTickerInfo.updateFundingTimestamp(new Date(Number(tickerMessage.nextFundingTime)))
pendingTickerInfo.updateFundingRate(Number(tickerMessage.fundingRate))
}
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
type BitgetTradeMessage = {
action: 'update'
arg: { instType: 'SPOT'; channel: 'trade'; instId: 'OPUSDT' }
data: [{ ts: '1730332800983'; price: '1.717'; size: '56.16'; side: 'buy'; tradeId: '1235670816495050754' }]
ts: 1730332800989
}
type BitgetOrderbookMessage = {
action: 'snapshot'
arg: { instType: 'SPOT'; channel: 'books15'; instId: 'GEMSUSDT' }
data: [
{
asks: [['0.22816', '155.25']]
bids: [['0.22785', '73.41']]
checksum: 0
ts: '1730963759993'
}
]
ts: 1730963759997
}
type BitgetBBoMessage = {
action: 'snapshot'
arg: { instType: 'SPOT'; channel: 'books1'; instId: 'METISUSDT' }
data: [{ asks: [['44.90', '0.6927']]; bids: [['44.82', '3.5344']]; checksum: 0; ts: '1730332859988' }]
ts: 1730332859989
}
type BitgetTickerMessage = {
action: 'snapshot'
arg: { instType: 'COIN-FUTURES'; channel: 'ticker'; instId: 'BTCUSD' }
data: [
{
instId: 'BTCUSD'
lastPr: '72331.5'
bidPr: '72331.5'
askPr: '72331.8'
bidSz: '7.296'
askSz: '0.02'
open24h: '72047.8'
high24h: '72934.8'
low24h: '71422.8'
change24h: '-0.00561'
fundingRate: '0.000116'
nextFundingTime: string
markPrice: string
indexPrice: string
holdingAmount: string
baseVolume: '7543.376'
quoteVolume: '544799876.924'
openUtc: '72335.3'
symbolType: '1'
symbol: 'BTCUSD'
deliveryPrice: '0'
ts: '1730332823217'
}
]
ts: 1730332823220
}
================================================
FILE: src/mappers/bitmex.ts
================================================
import { asNumberIfValid, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookPriceLevel, BookTicker, DerivativeTicker, FilterForExchange, Liquidation, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://www.bitmex.com/app/wsAPI
export const bitmexTradesMapper: Mapper<'bitmex', Trade> = {
canHandle(message: BitmexDataMessage) {
return message.table === 'trade' && message.action === 'insert'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
}
]
},
*map(bitmexTradesMessage: BitmexTradesMessage, localTimestamp: Date) {
for (const bitmexTrade of bitmexTradesMessage.data) {
const trade: Trade = {
type: 'trade',
symbol: bitmexTrade.symbol,
exchange: 'bitmex',
id: bitmexTrade.trdMatchID,
price: bitmexTrade.price,
amount: bitmexTrade.size,
side: bitmexTrade.side !== undefined ? (bitmexTrade.side === 'Buy' ? 'buy' : 'sell') : 'unknown',
timestamp: new Date(bitmexTrade.timestamp),
localTimestamp: localTimestamp
}
yield trade
}
}
}
export class BitmexBookChangeMapper implements Mapper<'bitmex', BookChange> {
private readonly _idToPriceLevelMap: Map = new Map()
canHandle(message: BitmexDataMessage) {
return message.table === 'orderBookL2'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'orderBookL2',
symbols
} as const
]
}
*map(bitmexOrderBookL2Message: BitmexOrderBookL2Message, localTimestamp: Date): IterableIterator {
let bitmexBookMessagesGrouppedBySymbol
// only partial messages can contain different symbols (when subscribed via {"op": "subscribe", "args": ["orderBookL2"]} for example)
if (bitmexOrderBookL2Message.action === 'partial') {
bitmexBookMessagesGrouppedBySymbol = bitmexOrderBookL2Message.data.reduce(
(prev, current) => {
if (prev[current.symbol]) {
prev[current.symbol].push(current)
} else {
prev[current.symbol] = [current]
}
return prev
},
{} as {
[key: string]: typeof bitmexOrderBookL2Message.data
}
)
if (bitmexOrderBookL2Message.data.length === 0 && bitmexOrderBookL2Message.filter?.symbol !== undefined) {
const emptySnapshot: BookChange = {
type: 'book_change',
symbol: bitmexOrderBookL2Message.filter?.symbol!,
exchange: 'bitmex',
isSnapshot: true,
bids: [],
asks: [],
timestamp: localTimestamp,
localTimestamp: localTimestamp
}
yield emptySnapshot
}
} else {
// in case of other messages types BitMEX always returns data for single symbol
bitmexBookMessagesGrouppedBySymbol = {
[bitmexOrderBookL2Message.data[0].symbol]: bitmexOrderBookL2Message.data
}
}
for (let symbol in bitmexBookMessagesGrouppedBySymbol) {
const bids: BookPriceLevel[] = []
const asks: BookPriceLevel[] = []
let latestBitmexTimestamp: Date | undefined = undefined
for (const item of bitmexBookMessagesGrouppedBySymbol[symbol]) {
if (item.timestamp !== undefined) {
const priceLevelTimestamp = new Date(item.timestamp)
if (latestBitmexTimestamp === undefined) {
latestBitmexTimestamp = priceLevelTimestamp
} else {
if (priceLevelTimestamp.valueOf() > latestBitmexTimestamp.valueOf()) {
latestBitmexTimestamp = priceLevelTimestamp
}
}
}
// https://www.bitmex.com/app/restAPI#OrderBookL2
if (item.price !== undefined) {
// store the mapping from id to price level if price is specified
// only partials and inserts have price set
this._idToPriceLevelMap.set(item.id, item.price)
}
const price = this._idToPriceLevelMap.get(item.id)
const amount = item.size || 0 // delete messages do not have size specified
// if we still don't have a price it means that there was an update before partial message - let's skip it
if (price === undefined) {
continue
}
if (item.side === 'Buy') {
bids.push({ price, amount })
} else {
asks.push({ price, amount })
}
// remove meta info for deleted level
if (bitmexOrderBookL2Message.action === 'delete') {
this._idToPriceLevelMap.delete(item.id)
}
}
const isSnapshot = bitmexOrderBookL2Message.action === 'partial'
if (bids.length > 0 || asks.length > 0 || isSnapshot) {
const bookChange: BookChange = {
type: 'book_change',
symbol,
exchange: 'bitmex',
isSnapshot,
bids,
asks,
timestamp: latestBitmexTimestamp !== undefined ? latestBitmexTimestamp : localTimestamp,
localTimestamp: localTimestamp
}
yield bookChange
}
}
}
}
export class BitmexDerivativeTickerMapper implements Mapper<'bitmex', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: BitmexDataMessage) {
return message.table === 'instrument'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'instrument',
symbols
} as const
]
}
*map(message: BitmexInstrumentsMessage, localTimestamp: Date): IterableIterator {
for (const bitmexInstrument of message.data) {
// process instrument messages only if:
// - we already have seen their 'partials' or already have 'pending info'
// - and instruments aren't settled or unlisted already
const isOpen = bitmexInstrument.state === undefined || bitmexInstrument.state === 'Open' || bitmexInstrument.state === 'Closed'
const isPartial = message.action === 'partial'
const hasPendingInfo = this.pendingTickerInfoHelper.hasPendingTickerInfo(bitmexInstrument.symbol)
if ((isPartial || hasPendingInfo) && isOpen) {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(bitmexInstrument.symbol, 'bitmex')
pendingTickerInfo.updateFundingRate(bitmexInstrument.fundingRate)
pendingTickerInfo.updatePredictedFundingRate(bitmexInstrument.indicativeFundingRate)
pendingTickerInfo.updateFundingTimestamp(
bitmexInstrument.fundingTimestamp ? new Date(bitmexInstrument.fundingTimestamp) : undefined
)
pendingTickerInfo.updateIndexPrice(bitmexInstrument.indicativeSettlePrice)
pendingTickerInfo.updateMarkPrice(bitmexInstrument.markPrice)
pendingTickerInfo.updateOpenInterest(bitmexInstrument.openInterest)
pendingTickerInfo.updateLastPrice(bitmexInstrument.lastPrice)
if (bitmexInstrument.timestamp !== undefined) {
pendingTickerInfo.updateTimestamp(new Date(bitmexInstrument.timestamp))
}
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
}
export const bitmexLiquidationsMapper: Mapper<'bitmex', Liquidation> = {
canHandle(message: BitmexDataMessage) {
return message.table === 'liquidation' && message.action === 'insert'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'liquidation',
symbols
}
]
},
*map(bitmexLiquiationsMessage: BitmexLiquidation, localTimestamp: Date) {
for (const bitmexLiquidation of bitmexLiquiationsMessage.data) {
const liquidation: Liquidation = {
type: 'liquidation',
symbol: bitmexLiquidation.symbol,
exchange: 'bitmex',
id: bitmexLiquidation.orderID,
price: bitmexLiquidation.price,
amount: bitmexLiquidation.leavesQty,
side: bitmexLiquidation.side === 'Buy' ? 'buy' : 'sell',
timestamp: localTimestamp,
localTimestamp: localTimestamp
}
yield liquidation
}
}
}
export const bitmexBookTickerMapper: Mapper<'bitmex', BookTicker> = {
canHandle(message: BitmexDataMessage) {
return message.table === 'quote' && message.action === 'insert'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'quote',
symbols
}
]
},
*map(bitmexQuoteMessage: BitmexQuote, localTimestamp: Date) {
for (const bitmexQuote of bitmexQuoteMessage.data) {
const ticker: BookTicker = {
type: 'book_ticker',
symbol: bitmexQuote.symbol,
exchange: 'bitmex',
askAmount: asNumberIfValid(bitmexQuote.askSize),
askPrice: asNumberIfValid(bitmexQuote.askPrice),
bidPrice: asNumberIfValid(bitmexQuote.bidPrice),
bidAmount: asNumberIfValid(bitmexQuote.bidSize),
timestamp: new Date(bitmexQuote.timestamp),
localTimestamp: localTimestamp
}
yield ticker
}
}
}
type BitmexDataMessage = {
table: FilterForExchange['bitmex']['channel']
action: 'partial' | 'update' | 'insert' | 'delete'
}
type BitmexTradesMessage = BitmexDataMessage & {
table: 'trade'
action: 'insert'
data: {
symbol: string
trdMatchID: string
side?: 'Buy' | 'Sell'
size: number
price: number
timestamp: string
}[]
}
type BitmexInstrument = {
symbol: string
state?: 'Open' | 'Closed' | 'Unlisted' | 'Settled'
openInterest?: number | null
fundingRate?: number | null
markPrice?: number | null
lastPrice?: number | null
indicativeSettlePrice?: number | null
indicativeFundingRate?: number | null
fundingTimestamp?: string | null
timestamp?: string
}
type BitmexInstrumentsMessage = BitmexDataMessage & {
table: 'instrument'
data: BitmexInstrument[]
}
type BitmexOrderBookL2Message = BitmexDataMessage & {
table: 'orderBookL2'
filter?: { symbol?: string }
data: {
symbol: string
id: number
side: 'Buy' | 'Sell'
size?: number
price?: number
timestamp?: string
}[]
}
type BitmexLiquidation = BitmexDataMessage & {
table: 'liquidation'
data: {
orderID: string
symbol: string
side: 'Buy' | 'Sell'
price: number
leavesQty: number
}[]
}
type BitmexQuote = BitmexDataMessage & {
table: 'quote'
action: 'insert'
data: [{ timestamp: string; symbol: string; bidSize: number; bidPrice: number; askPrice: number; askSize: number }]
}
================================================
FILE: src/mappers/bitnomial.ts
================================================
import { parseμs, upperCaseSymbols } from '../handy.ts'
import { BookChange, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
export const bitnomialTradesMapper: Mapper<'bitnomial', Trade> = {
canHandle(message: BitnomialTrade) {
return message.type === 'trade'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
}
]
},
*map(message: BitnomialTrade, localTimestamp: Date): IterableIterator {
const timestamp = new Date(message.timestamp)
timestamp.μs = parseμs(message.timestamp)
yield {
type: 'trade',
symbol: message.symbol,
exchange: 'bitnomial',
id: String(message.ack_id),
price: Number(message.price),
amount: Number(message.quantity),
side: message.taker_side === 'Bid' ? 'buy' : 'sell',
timestamp,
localTimestamp: localTimestamp
}
}
}
const mapBookLevel = (level: BookLevel) => {
return { price: level[0], amount: level[1] }
}
export class BitnomialBookChangMapper implements Mapper<'bitnomial', BookChange> {
canHandle(message: BitnomialBookMessage) {
return message.type === 'book' || message.type === 'level'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'book',
symbols
} as const,
{
channel: 'level',
symbols
} as const
]
}
*map(message: BitnomialBookMessage, localTimestamp: Date): IterableIterator {
const timestamp = new Date(message.timestamp)
timestamp.μs = parseμs(message.timestamp)
if (message.type === 'book') {
yield {
type: 'book_change',
symbol: message.symbol,
exchange: 'bitnomial',
isSnapshot: true,
bids: message.bids.map(mapBookLevel),
asks: message.asks.map(mapBookLevel),
timestamp,
localTimestamp
}
} else {
const update = {
price: message.price,
amount: message.quantity
}
yield {
type: 'book_change',
symbol: message.symbol,
exchange: 'bitnomial',
isSnapshot: false,
bids: message.side === 'Bid' ? [update] : [],
asks: message.side === 'Ask' ? [update] : [],
timestamp,
localTimestamp: localTimestamp
}
}
}
}
type BitnomialTrade = {
type: 'trade'
ack_id: '7148460953766461527'
price: 19000
quantity: 10
symbol: 'BUSZ2'
taker_side: 'Bid'
timestamp: '2022-09-28T16:06:39.022836179Z'
}
type BookLevel = [number, number]
type BitnomialBookMessage =
| {
ack_id: '7187577067767395971'
price: 18970
quantity: 5
side: 'Ask' | 'Bid'
symbol: 'BUIH23'
timestamp: '2023-01-12T20:03:28.292532617Z'
type: 'level'
}
| {
ack_id: '7187577067767395784'
asks: BookLevel[]
bids: BookLevel[]
symbol: 'BUSH23'
timestamp: '2023-01-12T20:03:06.479197763Z'
type: 'book'
}
================================================
FILE: src/mappers/bitstamp.ts
================================================
import { lowerCaseSymbols } from '../handy.ts'
import { BookChange, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
// https://www.bitstamp.net/websocket/v2/
export const bitstampTradesMapper: Mapper<'bitstamp', Trade> = {
canHandle(message: BitstampTrade | BitstampDiffOrderBook | BitstampDiffOrderBookSnapshot) {
if (message.data === undefined) {
return false
}
return message.channel.startsWith('live_trades') && message.event === 'trade'
},
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'live_trades',
symbols
}
]
},
*map(bitstampTradeResponse: BitstampTrade, localTimestamp: Date): IterableIterator {
const bitstampTrade = bitstampTradeResponse.data
const symbol = bitstampTradeResponse.channel.slice(bitstampTradeResponse.channel.lastIndexOf('_') + 1)
const microtimestamp = Number(bitstampTrade.microtimestamp)
const timestamp = new Date(microtimestamp / 1000)
timestamp.μs = microtimestamp % 1000
yield {
type: 'trade',
symbol: symbol.toUpperCase(),
exchange: 'bitstamp',
id: String(bitstampTrade.id),
price: Number(bitstampTrade.price),
amount: Number(bitstampTrade.amount),
side: bitstampTrade.type === 0 ? 'buy' : 'sell',
timestamp,
localTimestamp
}
}
}
export class BitstampBookChangeMapper implements Mapper<'bitstamp', BookChange> {
private readonly _symbolToDepthInfoMapping: { [key: string]: LocalDepthInfo } = {}
canHandle(message: BitstampTrade | BitstampDiffOrderBook | BitstampDiffOrderBookSnapshot) {
if (message.data === undefined) {
return false
}
return message.channel.startsWith('diff_order_book') && (message.event === 'data' || message.event === 'snapshot')
}
getFilters(symbols?: string[]) {
symbols = lowerCaseSymbols(symbols)
return [
{
channel: 'diff_order_book',
symbols
} as const
]
}
*map(message: BitstampDiffOrderBookSnapshot | BitstampDiffOrderBook, localTimestamp: Date): IterableIterator {
const symbol = message.channel.slice(message.channel.lastIndexOf('_') + 1).toUpperCase()
if (this._symbolToDepthInfoMapping[symbol] === undefined) {
this._symbolToDepthInfoMapping[symbol] = {
bufferedUpdates: []
}
}
const symbolDepthInfo = this._symbolToDepthInfoMapping[symbol]
const snapshotAlreadyProcessed = symbolDepthInfo.snapshotProcessed
// first check if received message is snapshot and process it as such if it is
if (message.event === 'snapshot') {
// produce snapshot book_change
let timestamp
if (message.data.microtimestamp !== undefined) {
const microtimestamp = Number(message.data.microtimestamp)
timestamp = new Date(microtimestamp / 1000)
timestamp.μs = microtimestamp % 1000
} else {
timestamp = new Date(Number(message.data.timestamp) * 1000)
}
yield {
type: 'book_change',
symbol,
exchange: 'bitstamp',
isSnapshot: true,
bids: message.data.bids !== undefined ? message.data.bids.map(this._mapBookLevel) : [],
asks: message.data.asks !== undefined ? message.data.asks.map(this._mapBookLevel) : [],
timestamp,
localTimestamp
}
// mark given symbol depth info that has snapshot processed
symbolDepthInfo.lastUpdateTimestamp = Number(message.data.timestamp)
if (message.data.microtimestamp !== undefined) {
symbolDepthInfo.lastUpdateMicroTimestamp = Number(message.data.microtimestamp)
}
symbolDepthInfo.snapshotProcessed = true
// if there were any depth updates buffered, let's proccess those
for (const update of symbolDepthInfo.bufferedUpdates) {
const bookChange = this._mapBookDepthUpdate(update, localTimestamp, symbolDepthInfo, symbol)
if (bookChange !== undefined) {
yield bookChange
}
}
// remove all buffered updates
symbolDepthInfo.bufferedUpdates = []
} else if (snapshotAlreadyProcessed) {
// snapshot was already processed let's map the message as normal book_change
const bookChange = this._mapBookDepthUpdate(message, localTimestamp, symbolDepthInfo, symbol)
if (bookChange !== undefined) {
yield bookChange
}
} else {
// if snapshot hasn't been yet processed and we've got depthUpdate message, let's buffer it for later processing
symbolDepthInfo.bufferedUpdates.push(message)
}
}
private _mapBookDepthUpdate(
bitstampBookUpdate: BitstampDiffOrderBook,
localTimestamp: Date,
depthInfo: LocalDepthInfo,
symbol: string
): BookChange | undefined {
const microtimestamp = Number(bitstampBookUpdate.data.microtimestamp)
// skip all book updates that preceed book snapshot
// REST API not always returned microtimestamps for initial order book snapshots
// fallback to timestamp
if (depthInfo.lastUpdateMicroTimestamp !== undefined && microtimestamp <= depthInfo.lastUpdateMicroTimestamp) {
return
} else if (Number(bitstampBookUpdate.data.timestamp) < depthInfo.lastUpdateTimestamp!) {
return
}
const timestamp = new Date(microtimestamp / 1000)
timestamp.μs = microtimestamp % 1000
return {
type: 'book_change',
symbol,
exchange: 'bitstamp',
isSnapshot: false,
bids: bitstampBookUpdate.data.bids !== undefined ? bitstampBookUpdate.data.bids.map(this._mapBookLevel) : [],
asks: bitstampBookUpdate.data.asks !== undefined ? bitstampBookUpdate.data.asks.map(this._mapBookLevel) : [],
timestamp: timestamp,
localTimestamp
}
}
private _mapBookLevel(level: BitstampBookLevel) {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
}
type BitstampTrade = {
event: 'trade'
channel: string
data: {
microtimestamp: string
amount: number
price: number
type: number
id: number
}
}
type BitstampBookLevel = [string, string]
type BitstampDiffOrderBook = {
data: {
microtimestamp: string
timestamp: string
bids: BitstampBookLevel[]
asks: BitstampBookLevel[]
}
event: 'data'
channel: string
}
type BitstampDiffOrderBookSnapshot = {
event: 'snapshot'
channel: string
data: {
timestamp: string
microtimestamp?: string
bids: BitstampBookLevel[]
asks: BitstampBookLevel[]
}
}
type LocalDepthInfo = {
bufferedUpdates: BitstampDiffOrderBook[]
snapshotProcessed?: boolean
lastUpdateTimestamp?: number
lastUpdateMicroTimestamp?: number
}
================================================
FILE: src/mappers/blockchaincom.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
export class BlockchainComTradesMapper implements Mapper<'blockchain-com', Trade> {
canHandle(message: BlockchainComTradeMessage) {
return message.channel === 'trades' && message.event === 'updated'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
} as const
]
}
*map(message: BlockchainComTradeMessage, localTimestamp: Date): IterableIterator {
yield {
type: 'trade',
symbol: message.symbol,
exchange: 'blockchain-com',
id: message.trade_id,
price: message.price,
amount: message.qty,
side: message.side === 'sell' ? 'sell' : 'buy',
timestamp: new Date(message.timestamp),
localTimestamp: localTimestamp
}
}
}
export class BlockchainComBookChangeMapper implements Mapper<'blockchain-com', BookChange> {
canHandle(message: BlockchainComL2Message) {
return message.channel == 'l2' && (message.event === 'snapshot' || message.event === 'updated')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'l2',
symbols
}
]
}
*map(message: BlockchainComL2Message, localTimestamp: Date): IterableIterator {
yield {
type: 'book_change',
symbol: message.symbol,
exchange: 'blockchain-com',
isSnapshot: message.event === 'snapshot',
bids: message.bids.map(this.mapBookLevel),
asks: message.asks.map(this.mapBookLevel),
timestamp: new Date(message.timestamp),
localTimestamp
}
}
protected mapBookLevel(level: { px: number; qty: number }) {
return { price: level.px, amount: level.qty }
}
}
type BlockchainComTradeMessage = {
seqnum: 408403
event: 'updated'
channel: 'trades'
symbol: 'ETH-USDT'
timestamp: '2023-02-23T03:02:11.503718Z'
side: 'sell'
qty: 0.60192856
price: 1677.94
trade_id: '844558083396024'
}
type BlockchainComL2Message =
| {
seqnum: 482554
event: 'updated'
channel: 'l2'
symbol: 'DOT-GBP'
bids: [{ num: 1; px: 6.08; qty: 137.77377093 }]
asks: []
timestamp: '2023-02-23T03:02:11.535015Z'
}
| {
seqnum: 269087
event: 'snapshot'
channel: 'l2'
symbol: 'BTC-USD'
bids: [{ num: 1; px: 1.8; qty: 7.45715496 }]
asks: [{ num: 1; px: 24187.8; qty: 0.04175659 }]
timestamp: '2023-02-23T00:00:00.127804Z'
}
================================================
FILE: src/mappers/bybit.ts
================================================
import { asNumberIfValid, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Exchange, Liquidation, OptionSummary, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// v5 https://bybit-exchange.github.io/docs/v5/ws/connect
export class BybitV5TradesMapper implements Mapper<'bybit' | 'bybit-spot' | 'bybit-options', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BybitV5Trade) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith('publicTrade.')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'publicTrade',
symbols
} as const
]
}
*map(message: BybitV5Trade, localTimestamp: Date): IterableIterator {
for (const trade of message.data) {
yield {
type: 'trade',
symbol: trade.s,
exchange: this._exchange,
id: trade.i,
price: Number(trade.p),
amount: Number(trade.v),
side: trade.S == 'Buy' ? 'buy' : trade.S === 'Sell' ? 'sell' : 'unknown',
timestamp: new Date(trade.T),
localTimestamp
}
}
}
}
export class BybitV5BookChangeMapper implements Mapper<'bybit' | 'bybit-spot' | 'bybit-options', BookChange> {
constructor(protected readonly _exchange: Exchange, private readonly _depth: number) {}
canHandle(message: BybitV5OrderBookMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith(`orderbook.${this._depth}.`)
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: `orderbook.${this._depth}`,
symbols
} as const
]
}
*map(message: BybitV5OrderBookMessage, localTimestamp: Date) {
yield {
type: 'book_change',
symbol: message.data.s,
exchange: this._exchange,
isSnapshot: message.type === 'snapshot',
bids: message.data.b.map(this._mapBookLevel),
asks: message.data.a.map(this._mapBookLevel),
timestamp: new Date(message.ts),
localTimestamp
} as const
}
private _mapBookLevel(level: [string, string]) {
return { price: Number(level[0]), amount: Number(level[1]) }
}
}
export class BybitV5BookTickerMapper implements Mapper<'bybit' | 'bybit-spot', BookTicker> {
private _snapshots: {
[key: string]: {
askAmount: number | undefined
askPrice: number | undefined
bidPrice: number | undefined
bidAmount: number | undefined
}
} = {}
constructor(protected readonly _exchange: Exchange) {}
canHandle(message: BybitV5OrderBookMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith(`orderbook.1.`)
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'orderbook.1',
symbols
} as const
]
}
*map(message: BybitV5OrderBookMessage, localTimestamp: Date) {
const bestAsk = message.data.a.filter((ask) => ask[1] != '0')[0]
const bestBid = message.data.b.filter((bid) => bid[1] != '0')[0]
if (message.type === 'snapshot') {
this._snapshots[message.data.s] = {
askAmount: bestAsk !== undefined ? Number(bestAsk[1]) : undefined,
askPrice: bestAsk !== undefined ? Number(bestAsk[0]) : undefined,
bidPrice: bestBid !== undefined ? Number(bestBid[0]) : undefined,
bidAmount: bestBid !== undefined ? Number(bestBid[1]) : undefined
}
}
const matchingSnapshot = this._snapshots[message.data.s]
if (!matchingSnapshot) {
return
}
const bookTicker: BookTicker = {
type: 'book_ticker',
symbol: message.data.s,
exchange: this._exchange,
askAmount: bestAsk !== undefined ? Number(bestAsk[1]) : matchingSnapshot.askAmount,
askPrice: bestAsk !== undefined ? Number(bestAsk[0]) : matchingSnapshot.askPrice,
bidPrice: bestBid !== undefined ? Number(bestBid[0]) : matchingSnapshot.bidPrice,
bidAmount: bestBid !== undefined ? Number(bestBid[1]) : matchingSnapshot.bidAmount,
timestamp: new Date(message.ts),
localTimestamp: localTimestamp
}
this._snapshots[message.data.s] = {
askAmount: bookTicker.askAmount,
askPrice: bookTicker.askPrice,
bidPrice: bookTicker.bidPrice,
bidAmount: bookTicker.bidAmount
}
yield bookTicker
}
}
export class BybitV5DerivativeTickerMapper implements Mapper<'bybit', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: BybitV5DerivTickerMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith('tickers.')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'tickers',
symbols
} as const
]
}
*map(message: BybitV5DerivTickerMessage, localTimestamp: Date): IterableIterator {
const instrumentInfo = message.data
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(instrumentInfo.symbol, 'bybit')
if (instrumentInfo.fundingRate !== undefined && instrumentInfo.fundingRate !== '') {
pendingTickerInfo.updateFundingRate(Number(instrumentInfo.fundingRate))
}
if (instrumentInfo.nextFundingTime !== undefined && instrumentInfo.nextFundingTime !== '') {
pendingTickerInfo.updateFundingTimestamp(new Date(Number(instrumentInfo.nextFundingTime)))
}
if (instrumentInfo.indexPrice !== undefined && instrumentInfo.indexPrice !== '') {
pendingTickerInfo.updateIndexPrice(Number(instrumentInfo.indexPrice))
}
if (instrumentInfo.markPrice !== undefined && instrumentInfo.markPrice !== '') {
pendingTickerInfo.updateMarkPrice(Number(instrumentInfo.markPrice))
}
if (instrumentInfo.openInterest !== undefined && instrumentInfo.openInterest !== '') {
pendingTickerInfo.updateOpenInterest(Number(instrumentInfo.openInterest))
}
if (instrumentInfo.lastPrice !== undefined && instrumentInfo.lastPrice !== '') {
pendingTickerInfo.updateLastPrice(Number(instrumentInfo.lastPrice))
}
pendingTickerInfo.updateTimestamp(new Date(message.ts))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
export class BybitV5LiquidationsMapper implements Mapper<'bybit', Liquidation> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BybitV5LiquidationMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith('liquidation.')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'liquidation',
symbols
} as const
]
}
*map(message: BybitV5LiquidationMessage, localTimestamp: Date): IterableIterator {
// from bybit telegram: When "side":"Buy", a long position was liquidated. Will fix the docs.
const bybitLiquidation = message.data
const liquidation: Liquidation = {
type: 'liquidation',
symbol: bybitLiquidation.symbol,
exchange: this._exchange,
id: undefined,
price: Number(bybitLiquidation.price),
amount: Number(bybitLiquidation.size),
side: bybitLiquidation.side == 'Buy' ? 'sell' : 'buy',
timestamp: new Date(message.ts),
localTimestamp
}
yield liquidation
}
}
export class BybitV5AllLiquidationsMapper implements Mapper<'bybit', Liquidation> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BybitV5AllLiquidationMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith('allLiquidation.')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'allLiquidation',
symbols
} as const
]
}
*map(message: BybitV5AllLiquidationMessage, localTimestamp: Date): IterableIterator {
for (const bybitLiquidation of message.data) {
const liquidation: Liquidation = {
type: 'liquidation',
symbol: bybitLiquidation.s,
exchange: this._exchange,
id: undefined,
price: Number(bybitLiquidation.p),
amount: Number(bybitLiquidation.v),
// from bybit docs Position side. Buy,Sell. When you receive a Buy update, this means that a long position has been liquidated
side: bybitLiquidation.S == 'Buy' ? 'sell' : 'buy',
timestamp: new Date(bybitLiquidation.T),
localTimestamp
}
yield liquidation
}
}
}
export class BybitV5OptionSummaryMapper implements Mapper<'bybit-options', OptionSummary> {
canHandle(message: BybitV5OptionTickerMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith('tickers.')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'tickers',
symbols
} as const
]
}
*map(message: BybitV5OptionTickerMessage, localTimestamp: Date) {
const symbolParts = message.data.symbol.split('-')
const isPut = symbolParts[3] === 'P'
const strikePrice = Number(symbolParts[2])
const expirationDate = new Date(symbolParts[1] + 'Z')
expirationDate.setUTCHours(8)
const optionSummary: OptionSummary = {
type: 'option_summary',
symbol: message.data.symbol,
exchange: 'bybit-options',
optionType: isPut ? 'put' : 'call',
strikePrice,
expirationDate,
bestBidPrice: asNumberIfValid(message.data.bidPrice),
bestBidAmount: asNumberIfValid(message.data.bidSize),
bestBidIV: asNumberIfValid(message.data.bidIv),
bestAskPrice: asNumberIfValid(message.data.askPrice),
bestAskAmount: asNumberIfValid(message.data.askSize),
bestAskIV: asNumberIfValid(message.data.askIv),
lastPrice: asNumberIfValid(message.data.lastPrice),
openInterest: asNumberIfValid(message.data.openInterest),
markPrice: asNumberIfValid(message.data.markPrice),
markIV: asNumberIfValid(message.data.markPriceIv),
delta: asNumberIfValid(message.data.delta),
gamma: asNumberIfValid(message.data.gamma),
vega: asNumberIfValid(message.data.vega),
theta: asNumberIfValid(message.data.theta),
rho: undefined,
underlyingPrice: asNumberIfValid(message.data.underlyingPrice),
underlyingIndex: '',
timestamp: new Date(message.ts),
localTimestamp: localTimestamp
}
yield optionSummary
}
}
// https://github.com/bybit-exchange/bybit-official-api-docs/blob/master/en/websocket.md
export class BybitTradesMapper implements Mapper<'bybit', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BybitDataMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith('trade.')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
} as const
]
}
*map(message: BybitTradeDataMessage, localTimestamp: Date): IterableIterator {
for (const trade of message.data) {
const timestamp =
'trade_time_ms' in trade
? new Date(Number(trade.trade_time_ms))
: 'tradeTimeMs' in trade
? new Date(Number(trade.tradeTimeMs))
: new Date(trade.timestamp)
yield {
type: 'trade',
symbol: trade.symbol,
exchange: this._exchange,
id: 'trade_id' in trade ? trade.trade_id : trade.tradeId,
price: Number(trade.price),
amount: trade.size,
side: trade.side == 'Buy' ? 'buy' : trade.side === 'Sell' ? 'sell' : 'unknown',
timestamp,
localTimestamp
}
}
}
}
export class BybitBookChangeMapper implements Mapper<'bybit', BookChange> {
constructor(protected readonly _exchange: Exchange, private readonly _canUseBook200Channel: boolean) {}
canHandle(message: BybitDataMessage) {
if (message.topic === undefined) {
return false
}
if (this._canUseBook200Channel) {
return message.topic.startsWith('orderBook_200.')
} else {
return message.topic.startsWith('orderBookL2_25.')
}
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
if (this._canUseBook200Channel) {
return [
{
channel: 'orderBook_200',
symbols
} as const
]
}
return [
{
channel: 'orderBookL2_25',
symbols
} as const
]
}
*map(message: BybitBookSnapshotDataMessage | BybitBookSnapshotUpdateMessage, localTimestamp: Date) {
const topicArray = message.topic.split('.')
const symbol = topicArray[topicArray.length - 1]
const data =
message.type === 'snapshot'
? 'order_book' in message.data
? message.data.order_book
: 'orderBook' in message.data
? message.data.orderBook
: message.data
: [...message.data.delete, ...message.data.update, ...message.data.insert]
const timestampBybit = message.timestamp_e6 !== undefined ? Number(message.timestamp_e6) : Number(message.timestampE6)
const timestamp = new Date(timestampBybit / 1000)
timestamp.μs = timestampBybit % 1000
yield {
type: 'book_change',
symbol,
exchange: this._exchange,
isSnapshot: message.type === 'snapshot',
bids: data.filter((d) => d.side === 'Buy').map(this._mapBookLevel),
asks: data.filter((d) => d.side === 'Sell').map(this._mapBookLevel),
timestamp,
localTimestamp
} as const
}
private _mapBookLevel(level: BybitBookLevel) {
return { price: Number(level.price), amount: level.size !== undefined ? level.size : 0 }
}
}
export class BybitDerivativeTickerMapper implements Mapper<'bybit', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: BybitDataMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith('instrument_info.')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'instrument_info',
symbols
} as const
]
}
*map(message: BybitInstrumentDataMessage, localTimestamp: Date): IterableIterator {
const instrumentInfo = 'symbol' in message.data ? message.data : message.data.update[0]
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(instrumentInfo.symbol, 'bybit')
const fundingRate = 'funding_rate_e6' in instrumentInfo ? instrumentInfo.funding_rate_e6 : instrumentInfo.fundingRateE6
pendingTickerInfo.updateFundingRate(fundingRate !== undefined ? Number(fundingRate) / 1000000 : undefined)
const predictedFundingRate =
'predicted_funding_rate_e6' in instrumentInfo ? instrumentInfo.predicted_funding_rate_e6 : instrumentInfo.predictedFundingRateE6
pendingTickerInfo.updatePredictedFundingRate(predictedFundingRate !== undefined ? Number(predictedFundingRate) / 1000000 : undefined)
const nextFundingTime = 'next_funding_time' in instrumentInfo ? instrumentInfo.next_funding_time : instrumentInfo.nextFundingTime
pendingTickerInfo.updateFundingTimestamp(
nextFundingTime !== undefined && new Date(nextFundingTime).valueOf() > 0 ? new Date(nextFundingTime) : undefined
)
if (instrumentInfo.index_price !== undefined) {
pendingTickerInfo.updateIndexPrice(Number(instrumentInfo.index_price))
} else if (instrumentInfo.indexPrice !== undefined) {
pendingTickerInfo.updateIndexPrice(Number(instrumentInfo.indexPrice))
} else if (instrumentInfo.index_price_e4 !== undefined) {
pendingTickerInfo.updateIndexPrice(Number(instrumentInfo.index_price_e4) / 10000)
} else if (instrumentInfo.indexPriceE4 !== undefined) {
pendingTickerInfo.updateIndexPrice(Number(instrumentInfo.indexPriceE4) / 10000)
}
if (instrumentInfo.mark_price !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(instrumentInfo.mark_price))
} else if (instrumentInfo.markPrice !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(instrumentInfo.markPrice))
} else if (instrumentInfo.mark_price_e4 !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(instrumentInfo.mark_price_e4) / 10000)
} else if (instrumentInfo.markPriceE4 !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(instrumentInfo.markPriceE4) / 10000)
}
if (instrumentInfo.open_interest !== undefined) {
pendingTickerInfo.updateOpenInterest(instrumentInfo.open_interest)
} else if (instrumentInfo.openInterestE8 !== undefined) {
pendingTickerInfo.updateOpenInterest(Number(instrumentInfo.openInterestE8) / 100000000)
} else if (instrumentInfo.open_interest_e8 !== undefined) {
pendingTickerInfo.updateOpenInterest(instrumentInfo.open_interest_e8 / 100000000)
}
if (instrumentInfo.last_price !== undefined) {
pendingTickerInfo.updateLastPrice(Number(instrumentInfo.last_price))
} else if (instrumentInfo.lastPrice !== undefined) {
pendingTickerInfo.updateLastPrice(Number(instrumentInfo.lastPrice))
} else if (instrumentInfo.last_price_e4 !== undefined) {
pendingTickerInfo.updateLastPrice(Number(instrumentInfo.last_price_e4) / 10000)
} else if (instrumentInfo.lastPriceE4) {
pendingTickerInfo.updateLastPrice(Number(instrumentInfo.lastPriceE4) / 10000)
}
if (message.timestamp_e6 !== undefined) {
const timestampBybit = Number(message.timestamp_e6)
const timestamp = new Date(timestampBybit / 1000)
timestamp.μs = timestampBybit % 1000
pendingTickerInfo.updateTimestamp(timestamp)
} else if (message.timestampE6 !== undefined) {
const timestampBybit = Number(message.timestampE6)
const timestamp = new Date(timestampBybit / 1000)
timestamp.μs = timestampBybit % 1000
pendingTickerInfo.updateTimestamp(timestamp)
} else {
pendingTickerInfo.updateTimestamp(new Date(instrumentInfo.updated_at))
}
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
export class BybitLiquidationsMapper implements Mapper<'bybit', Liquidation> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BybitDataMessage) {
if (message.topic === undefined) {
return false
}
return message.topic.startsWith('liquidation.')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'liquidation',
symbols
} as const
]
}
*map(message: BybitLiquidationMessage | BybitLiquidationNativeMessage, localTimestamp: Date): IterableIterator {
// from bybit telegram: When "side":"Buy", a long position was liquidated. Will fix the docs.
if (message.generated) {
for (const bybitLiquidation of message.data) {
const liquidation: Liquidation = {
type: 'liquidation',
symbol: bybitLiquidation.symbol,
exchange: this._exchange,
id: String(bybitLiquidation.id),
price: Number(bybitLiquidation.price),
amount: bybitLiquidation.qty,
side: bybitLiquidation.side == 'Buy' ? 'sell' : 'buy',
timestamp: new Date(bybitLiquidation.time),
localTimestamp
}
yield liquidation
}
} else {
const bybitLiquidation = message.data
const liquidation: Liquidation = {
type: 'liquidation',
symbol: bybitLiquidation.symbol,
exchange: this._exchange,
id: undefined,
price: Number(bybitLiquidation.price),
amount: Number(bybitLiquidation.qty),
side: bybitLiquidation.side == 'Buy' ? 'sell' : 'buy',
timestamp: new Date(bybitLiquidation.time),
localTimestamp
}
yield liquidation
}
}
}
type BybitV5Trade =
| {
topic: 'publicTrade.LTCUSDT'
type: 'snapshot'
ts: 1680688979985
data: [
{
T: 1680688979983
s: 'LTCUSDT'
S: 'Buy'
v: '0.4'
p: '94.53'
L: 'ZeroMinusTick'
i: '4c7b6bdc-b4a3-5716-9c7b-bbe01dc7072f'
BT: false
}
]
}
| {
topic: 'publicTrade.BTCUSDC'
ts: 1680688980000
type: 'snapshot'
data: [{ i: '2240000000041223438'; T: 1680688979998; p: '28528.98'; v: '0.00433'; S: 'Buy'; s: 'BTCUSDC'; BT: false }]
}
| {
id: 'publicTrade.BTC-3414637898-1680652922102'
topic: 'publicTrade.BTC'
ts: 1680652922102
data: [
{ p: '985'; v: '0.01'; i: '0404c393-8419-5bac-95c3-5fea28404754'; T: 1680652922081; BT: false; s: 'BTC-28APR23-29500-C'; S: 'Sell' }
]
type: 'snapshot'
}
type BybitV5OrderBookMessage = {
topic: 'orderbook.50.LTCUSD'
type: 'snapshot' | 'delta'
ts: 1680673822478
data: {
s: string
b: [string, string][]
a: [string, string][]
u: 11802648
seq: 941860281
}
}
type BybitV5DerivTickerMessage = {
topic: 'tickers.BTCUSD'
type: 'snapshot' | 'delta'
data: {
symbol: string
lastPrice?: string
markPrice?: string
indexPrice?: string
openInterest?: string
openInterestValue?: string
nextFundingTime?: string
fundingRate?: string
bid1Price?: string
bid1Size?: string
ask1Price?: string
ask1Size?: string
}
cs: 20856433578
ts: 1680673822577
}
type BybitV5LiquidationMessage = {
data: {
price: '0.03803'
side: 'Buy'
size: '1637'
symbol: 'GALAUSDT'
updatedTime: 1673251091822
}
topic: 'liquidation.GALAUSDT'
ts: 1673251091822
type: 'snapshot'
}
type BybitV5AllLiquidationMessage = {
topic: 'allLiquidation.KAITOUSDT'
type: 'snapshot'
ts: 1740480190078
data: [{ T: 1740480189987; s: 'KAITOUSDT'; S: 'Buy'; v: '43'; p: '1.7531' }]
}
type BybitV5OptionTickerMessage = {
id: 'tickers.ETH-30JUN23-200-P-3164908233-1680652859919'
topic: 'tickers.ETH-30JUN23-200-P'
ts: 1680652859919
data: {
symbol: 'ETH-30JUN23-200-P'
bidPrice: '0.1'
bidSize: '5'
bidIv: '1.4744'
askPrice: '0'
askSize: '0'
askIv: '0'
lastPrice: '1'
highPrice24h: '0'
lowPrice24h: '0'
markPrice: '0.2548522'
indexPrice: '1871.27'
markPriceIv: '1.5991'
underlyingPrice: '1886.16'
openInterest: '231.5'
turnover24h: '0'
volume24h: '0'
totalVolume: '232'
totalTurnover: '362305'
delta: '-0.00052953'
gamma: '0.00000128'
vega: '0.01719155'
theta: '-0.0159208'
predictedDeliveryPrice: '0'
change24h: '0'
}
type: 'snapshot'
}
type BybitDataMessage = {
topic: string
}
type BybitTradeDataMessage =
| (BybitDataMessage & {
data: {
timestamp: string
trade_time_ms?: number | string
symbol: string
side: 'Buy' | 'Sell'
size: number
price: number | string
trade_id: string
}[]
})
| {
topic: 'trade.BTCPERP'
data: [
{
symbol: 'BTCPERP'
tickDirection: 'PlusTick'
price: '21213.00'
size: 0.007
timestamp: '2022-06-21T09:36:58.000Z'
tradeTimeMs: '1655804218524'
side: 'Sell'
tradeId: '7aad7741-f763-5f78-bf43-c38b29a40f67'
}
]
}
type BybitBookLevel = {
price: string
side: 'Buy' | 'Sell'
size?: number
}
type BybitBookSnapshotDataMessage = BybitDataMessage & {
type: 'snapshot'
data: BybitBookLevel[] | { order_book: BybitBookLevel[] } | { orderBook: BybitBookLevel[] }
timestamp_e6: number | string
timestampE6: number | string
}
type BybitBookSnapshotUpdateMessage = BybitDataMessage & {
type: 'delta'
data: {
delete: BybitBookLevel[]
update: BybitBookLevel[]
insert: BybitBookLevel[]
}
timestamp_e6: number | string
timestampE6: number | string
}
type BybitInstrumentUpdate = {
symbol: string
mark_price_e4?: number
mark_price?: string
index_price_e4?: string
index_price?: string
open_interest?: number
open_interest_e8?: number
funding_rate_e6?: string
predicted_funding_rate_e6?: number
next_funding_time?: string
last_price_e4?: string
last_price?: string
updated_at: string
lastPriceE4: '212130000'
lastPrice: '21213.00'
lastTickDirection: 'PlusTick'
prevPrice24hE4: '207180000'
prevPrice24h: '20718.00'
price24hPcntE6: '23892'
highPrice24hE4: '214085000'
highPrice24h: '21408.50'
lowPrice24hE4: '198005000'
lowPrice24h: '19800.50'
prevPrice1hE4: '213315000'
prevPrice1h: '21331.50'
price1hPcntE6: '-5555'
markPriceE4: '212094700'
markPrice: '21209.47'
indexPriceE4: '212247200'
indexPrice: '21224.72'
openInterestE8: '18317600000'
totalTurnoverE8: '94568739311650000'
turnover24hE8: '1375880657550000'
totalVolumeE8: '2734659400000'
volume24hE8: '66536799999'
fundingRateE6: '-900'
predictedFundingRateE6: '-614'
crossSeq: '385207672'
createdAt: '1970-01-01T00:00:00.000Z'
updatedAt: '2022-06-21T09:36:58.000Z'
nextFundingTime: '2022-06-21T16:00:00Z'
countDownHour: '7'
bid1PriceE4: '212130000'
bid1Price: '21213.00'
ask1PriceE4: '212135000'
ask1Price: '21213.50'
}
type BybitInstrumentDataMessage =
| BybitDataMessage & {
timestamp_e6: string
timestampE6: string
data:
| BybitInstrumentUpdate
| {
update: [BybitInstrumentUpdate]
}
}
type BybitLiquidationMessage = BybitDataMessage & {
generated: true
data: {
id: number
qty: number
side: 'Sell' | 'Buy'
time: number
symbol: string
price: number
}[]
}
type BybitLiquidationNativeMessage = BybitDataMessage & {
generated: undefined
data: { symbol: string; side: 'Sell' | 'Buy'; price: string; qty: string; time: number }
}
================================================
FILE: src/mappers/bybitspot.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, Exchange, BookTicker, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
export class BybitSpotTradesMapper implements Mapper<'bybit-spot', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: BybitSpotTradeMessage) {
return message.topic === 'trade' && message.data !== undefined
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
} as const
]
}
*map(message: BybitSpotTradeMessage, localTimestamp: Date): IterableIterator {
const bybitTrade = message.data
yield {
type: 'trade',
symbol: message.params.symbol,
exchange: this._exchange,
id: bybitTrade.v,
price: Number(bybitTrade.p),
amount: Number(bybitTrade.q),
side: bybitTrade.m === true ? 'buy' : 'sell',
timestamp: new Date(bybitTrade.t),
localTimestamp
}
}
}
export class BybitSpotBookChangeMapper implements Mapper<'bybit-spot', BookChange> {
constructor(protected readonly _exchange: Exchange) {}
canHandle(message: BybitSpotDepthMessage) {
return message.topic === 'depth' && message.data !== undefined
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'depth',
symbols
} as const
]
}
*map(message: BybitSpotDepthMessage, localTimestamp: Date) {
yield {
type: 'book_change',
symbol: message.params.symbol,
exchange: this._exchange,
isSnapshot: true,
bids: message.data.b.map(this._mapBookLevel),
asks: message.data.a.map(this._mapBookLevel),
timestamp: new Date(message.data.t),
localTimestamp
} as const
}
private _mapBookLevel(level: [string, string]) {
return { price: Number(level[0]), amount: Number(level[1]) }
}
}
export class BybitSpotBookTickerMapper implements Mapper<'bybit-spot', BookTicker> {
constructor(protected readonly _exchange: Exchange) {}
canHandle(message: BybitSpotBookTickerMessage) {
return message.topic === 'bookTicker' && message.data !== undefined
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'bookTicker',
symbols
} as const
]
}
*map(message: BybitSpotBookTickerMessage, localTimestamp: Date) {
const bookTicker: BookTicker = {
type: 'book_ticker',
symbol: message.params.symbol,
exchange: this._exchange,
askAmount: message.data.askQty !== undefined ? Number(message.data.askQty) : undefined,
askPrice: message.data.askPrice !== undefined ? Number(message.data.askPrice) : undefined,
bidPrice: message.data.bidPrice !== undefined ? Number(message.data.bidPrice) : undefined,
bidAmount: message.data.bidQty !== undefined ? Number(message.data.bidQty) : undefined,
timestamp: new Date(message.data.time),
localTimestamp: localTimestamp
}
yield bookTicker
}
}
type BybitSpotBookTickerMessage = {
topic: 'bookTicker'
params: { symbol: 'BATUSDT'; binary: 'false'; symbolName: 'BATUSDT' }
data: { symbol: 'BATUSDT'; bidPrice: '0.3985'; bidQty: '1919.99'; askPrice: '0.3997'; askQty: '3747.68'; time: 1659311999973 }
}
type BybitSpotTradeMessage = {
topic: 'trade'
params: { symbol: 'XRP3SUSDT'; binary: 'false'; symbolName: 'XRP3SUSDT' }
data: { v: '2220000000006443832'; t: 1659312000387; p: '6.3957'; q: '3.5962'; m: boolean }
}
type BybitSpotDepthMessage = {
topic: 'depth'
params: { symbol: 'RENUSDT'; binary: 'false'; symbolName: 'RENUSDT' }
data: {
s: 'RENUSDT'
t: 1659312000390
v: '170667316_8244371_5'
b: [['0.14348', '3249.63']]
a: [['0.14457', '95.23']]
}
}
================================================
FILE: src/mappers/coinbase.ts
================================================
import { parseμs, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookPriceLevel, BookTicker, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
// https://docs.pro.coinbase.com/#websocket-feed
export const coinbaseTradesMapper: Mapper<'coinbase', Trade> = {
canHandle(message: CoinbaseTrade | CoinbaseLevel2Snapshot | CoinbaseLevel2Update) {
return message.type === 'match'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'match',
symbols
}
]
},
*map(message: CoinbaseTrade, localTimestamp: Date): IterableIterator {
const timestamp = new Date(message.time)
timestamp.μs = parseμs(message.time)
yield {
type: 'trade',
symbol: message.product_id,
exchange: 'coinbase',
id: String(message.trade_id),
price: Number(message.price),
amount: Number(message.size),
side: message.side === 'sell' ? 'buy' : 'sell', // coinbase side field indicates the maker order side
timestamp,
localTimestamp: localTimestamp
}
}
}
const mapUpdateBookLevel = (level: CoinbaseUpdateBookLevel) => {
const price = Number(level[1])
const amount = Number(level[2])
return { price, amount }
}
const mapSnapshotBookLevel = (level: CoinbaseSnapshotBookLevel) => {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
const validAmountsOnly = (level: BookPriceLevel) => {
if (Number.isNaN(level.amount)) {
return false
}
if (level.amount < 0) {
return false
}
return true
}
export class CoinbaseBookChangMapper implements Mapper<'coinbase', BookChange> {
private readonly _symbolLastTimestampMap = new Map()
canHandle(message: CoinbaseTrade | CoinbaseLevel2Snapshot | CoinbaseLevel2Update) {
return message.type === 'l2update' || message.type === 'snapshot'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'snapshot',
symbols
} as const,
{
channel: 'l2update',
symbols
} as const
]
}
*map(message: CoinbaseLevel2Update | CoinbaseLevel2Snapshot, localTimestamp: Date): IterableIterator {
if (message.type === 'snapshot') {
let timestamp
if (message.time !== undefined) {
timestamp = new Date(message.time)
if (timestamp.valueOf() < 0) {
timestamp = localTimestamp
} else {
timestamp.μs = parseμs(message.time)
}
} else {
timestamp = localTimestamp
}
yield {
type: 'book_change',
symbol: message.product_id,
exchange: 'coinbase',
isSnapshot: true,
bids: message.bids.map(mapSnapshotBookLevel).filter(validAmountsOnly),
asks: message.asks.map(mapSnapshotBookLevel).filter(validAmountsOnly),
timestamp,
localTimestamp
}
} else {
// in very rare cases, Coinbase was returning timestamps that aren't valid, like: "time":"0001-01-01T00:00:00.000000Z"
// but l2update message was still valid and we need to process it, in such case use timestamp of previous message
let timestamp = new Date(message.time)
if (timestamp.valueOf() < 0) {
let previousValidTimestamp = this._symbolLastTimestampMap.get(message.product_id)
if (previousValidTimestamp === undefined) {
return
}
timestamp = previousValidTimestamp
} else {
timestamp.μs = parseμs(message.time)
this._symbolLastTimestampMap.set(message.product_id, timestamp)
}
yield {
type: 'book_change',
symbol: message.product_id,
exchange: 'coinbase',
isSnapshot: false,
bids: message.changes.filter((c) => c[0] === 'buy').map(mapUpdateBookLevel),
asks: message.changes.filter((c) => c[0] === 'sell').map(mapUpdateBookLevel),
timestamp,
localTimestamp: localTimestamp
}
}
}
}
export const coinbaseBookTickerMapper: Mapper<'coinbase', BookTicker> = {
canHandle(message: CoinbaseTicker) {
return message.type === 'ticker'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'ticker',
symbols
}
]
},
*map(message: CoinbaseTicker, localTimestamp: Date): IterableIterator {
let timestamp = new Date(message.time)
if (message.time === undefined || timestamp.valueOf() < 0) {
timestamp = localTimestamp
} else {
timestamp.μs = parseμs(message.time)
}
yield {
type: 'book_ticker',
symbol: message.product_id,
exchange: 'coinbase',
askAmount: message.best_ask_size !== undefined ? Number(message.best_ask_size) : undefined,
askPrice: message.best_ask !== undefined ? Number(message.best_ask) : undefined,
bidPrice: message.best_bid !== undefined ? Number(message.best_bid) : undefined,
bidAmount: message.best_bid_size !== undefined ? Number(message.best_bid_size) : undefined,
timestamp,
localTimestamp: localTimestamp
}
}
}
type CoinbaseTrade = {
type: 'match'
trade_id: number
time: string
product_id: string
size: string
price: string
side: 'sell' | 'buy'
}
type CoinbaseSnapshotBookLevel = [string, string]
type CoinbaseLevel2Snapshot = {
type: 'snapshot'
product_id: string
bids: CoinbaseSnapshotBookLevel[]
asks: CoinbaseSnapshotBookLevel[]
time?: string
}
type CoinbaseUpdateBookLevel = ['buy' | 'sell', string, string]
type CoinbaseLevel2Update = {
type: 'l2update'
product_id: string
time: string
changes: CoinbaseUpdateBookLevel[]
}
type CoinbaseTicker =
| {
type: 'ticker'
sequence: 2349290585
product_id: 'CGLD-USD'
price: '5.415'
best_bid: '5.4149'
best_ask: '5.4150'
time: '2021-10-13T07:05:00.028961Z'
best_bid_size: undefined
best_ask_size: undefined
}
| {
type: 'ticker'
sequence: 50978628538
product_id: 'BTC-USD'
price: '17165.16'
open_24h: '16437.94'
volume_24h: '42492.05081975'
low_24h: '16423.37'
high_24h: '17259.37'
volume_30d: '1093827.95195495'
best_bid: '17165.15'
best_bid_size: '0.61540890'
best_ask: '17167.76'
best_ask_size: '0.18528568'
side: 'sell'
time: '2022-12-01T00:00:00.122581Z'
trade_id: 463751434
last_size: '0.05'
}
================================================
FILE: src/mappers/coinbaseinternational.ts
================================================
import { addMinutes, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookPriceLevel, BookTicker, DerivativeTicker, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
export const coinbaseInternationalTradesMapper: Mapper<'coinbase-international', Trade> = {
canHandle(message: CoinbaseInternationalTradeMessage) {
return message.channel === 'MATCH' && message.type === 'UPDATE'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'MATCH',
symbols
}
]
},
*map(message: CoinbaseInternationalTradeMessage, localTimestamp: Date): IterableIterator {
yield {
type: 'trade',
symbol: message.product_id,
exchange: 'coinbase-international',
id: message.match_id,
price: Number(message.trade_price),
amount: Number(message.trade_qty),
side: message.aggressor_side === 'SELL' ? 'sell' : message.aggressor_side === 'BUY' ? 'buy' : 'unknown',
timestamp: new Date(message.time),
localTimestamp: localTimestamp
}
}
}
const mapUpdateBookLevel = (level: CoinbaseInternationalUpdateBookLevel) => {
const price = Number(level[1])
const amount = Number(level[2])
return { price, amount }
}
const mapSnapshotBookLevel = (level: CoinbaseInternationalSnapshotBookLevel) => {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
const validAmountsOnly = (level: BookPriceLevel) => {
if (Number.isNaN(level.amount)) {
return false
}
if (level.amount < 0) {
return false
}
return true
}
export class CoinbaseInternationalBookChangMapper implements Mapper<'coinbase-international', BookChange> {
canHandle(message: CoinbaseInternationalLevel2Snapshot | CoinbaseInternationalLevel2Update) {
return message.channel === 'LEVEL2' && (message.type === 'SNAPSHOT' || message.type === 'UPDATE')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'LEVEL2',
symbols
} as const
]
}
*map(
message: CoinbaseInternationalLevel2Snapshot | CoinbaseInternationalLevel2Update,
localTimestamp: Date
): IterableIterator {
if (message.type === 'SNAPSHOT') {
let timestamp
if (message.time !== undefined) {
timestamp = new Date(message.time)
if (timestamp.valueOf() < 0) {
timestamp = localTimestamp
}
} else {
timestamp = localTimestamp
}
yield {
type: 'book_change',
symbol: message.product_id,
exchange: 'coinbase-international',
isSnapshot: true,
bids: message.bids.map(mapSnapshotBookLevel).filter(validAmountsOnly),
asks: message.asks.map(mapSnapshotBookLevel).filter(validAmountsOnly),
timestamp,
localTimestamp
}
} else {
let timestamp = new Date(message.time)
yield {
type: 'book_change',
symbol: message.product_id,
exchange: 'coinbase-international',
isSnapshot: false,
bids: message.changes.filter((c) => c[0] === 'BUY').map(mapUpdateBookLevel),
asks: message.changes.filter((c) => c[0] === 'SELL').map(mapUpdateBookLevel),
timestamp,
localTimestamp: localTimestamp
}
}
}
}
export const coinbaseInternationalBookTickerMapper: Mapper<'coinbase-international', BookTicker> = {
canHandle(message: CoinbaseInternationalLevel1Message) {
return message.channel === 'LEVEL1' && (message.type === 'SNAPSHOT' || message.type === 'UPDATE')
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'LEVEL1',
symbols
}
]
},
*map(message: CoinbaseInternationalLevel1Message, localTimestamp: Date): IterableIterator {
let timestamp = new Date(message.time)
if (message.time === undefined || timestamp.valueOf() < 0) {
timestamp = localTimestamp
}
yield {
type: 'book_ticker',
symbol: message.product_id,
exchange: 'coinbase-international',
askAmount: message.ask_qty !== undefined ? Number(message.ask_qty) : undefined,
askPrice: message.ask_price !== undefined ? Number(message.ask_price) : undefined,
bidPrice: message.bid_price !== undefined ? Number(message.bid_price) : undefined,
bidAmount: message.bid_qty !== undefined ? Number(message.bid_qty) : undefined,
timestamp,
localTimestamp: localTimestamp
}
}
}
export class CoinbaseInternationalDerivativeTickerMapper implements Mapper<'coinbase-international', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: CoinbaseInternationalTradeMessage | CoinbaseInternationalRiskMessage | CoinbaseInternationalFundingMessage) {
// perps only
if (message.product_id === undefined || message.product_id.endsWith('-PERP') === false) {
return false
}
if (message.channel === 'MATCH' && message.type === 'UPDATE') {
return true
}
if (message.channel === 'FUNDING' && message.type === 'UPDATE') {
return true
}
if (message.channel === 'RISK') {
return true
}
return false
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'MATCH',
symbols
} as const,
{
channel: 'RISK',
symbols
} as const,
{
channel: 'FUNDING',
symbols
} as const
]
}
*map(
message: CoinbaseInternationalTradeMessage | CoinbaseInternationalRiskMessage | CoinbaseInternationalFundingMessage,
localTimestamp: Date
): IterableIterator {
if (message.channel === 'MATCH') {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.product_id, 'coinbase-international')
pendingTickerInfo.updateLastPrice(Number(message.trade_price))
return
}
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.product_id, 'coinbase-international')
if (message.channel === 'RISK') {
if (message.index_price !== undefined) {
pendingTickerInfo.updateIndexPrice(Number(message.index_price))
}
if (message.mark_price !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(message.mark_price))
}
if (message.open_interest !== undefined) {
pendingTickerInfo.updateOpenInterest(Number(message.open_interest))
}
}
if (message.channel === 'FUNDING') {
let nextFundingTime = new Date(message.time)
if (message.is_final === false) {
// If the field is_final is false, the message indicates the predicted funding rate for the next funding interval.
// https://docs.cdp.coinbase.com/intx/docs/websocket-channels#funding-channel
nextFundingTime.setUTCMinutes(0, 0, 0)
nextFundingTime = addMinutes(nextFundingTime, 60)
pendingTickerInfo.updateFundingTimestamp(nextFundingTime)
}
pendingTickerInfo.updateFundingRate(Number(message.funding_rate))
}
pendingTickerInfo.updateTimestamp(new Date(message.time))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
// TODO: real-time
type CoinbaseInternationalTradeMessage = {
sequence: 80
match_id: '374491377330814981'
trade_price: '0.009573'
trade_qty: '1651'
aggressor_side: 'BUY' | 'SELL' | 'OPENING_FILL'
channel: 'MATCH'
type: 'UPDATE'
time: '2024-10-30T10:55:02.069Z'
product_id: 'MEW-PERP'
}
type CoinbaseInternationalSnapshotBookLevel = [string, string]
type CoinbaseInternationalLevel2Snapshot = {
sequence: 81053126
bids: CoinbaseInternationalSnapshotBookLevel[]
asks: CoinbaseInternationalSnapshotBookLevel[]
channel: 'LEVEL2'
type: 'SNAPSHOT'
time: '2024-11-06T23:59:59.812Z'
product_id: 'BB-PERP'
}
type CoinbaseInternationalUpdateBookLevel = ['BUY' | 'SELL', string, string]
type CoinbaseInternationalLevel2Update = {
sequence: 162
changes: CoinbaseInternationalUpdateBookLevel[]
channel: 'LEVEL2'
type: 'UPDATE'
time: '2024-10-30T10:55:02.348Z'
product_id: 'NOT-PERP'
}
type CoinbaseInternationalLevel1Message =
| {
sequence: 65960075
bid_price: '27.03'
bid_qty: '24.404'
ask_price: '27.037'
ask_qty: '32.302'
channel: 'LEVEL1'
type: 'SNAPSHOT'
time: '2024-11-07T00:00:00.121Z'
product_id: 'AVAX-PERP'
}
| {
sequence: 120100774
bid_price: '2719.96'
bid_qty: '0.3676'
ask_price: '2720.25'
ask_qty: '0.919'
channel: 'LEVEL1'
type: 'UPDATE'
time: '2024-11-07T00:00:59.979Z'
product_id: 'ETH-USDC'
}
type CoinbaseInternationalRiskMessage = {
sequence: 108523490
limit_up: '0.5107'
limit_down: '0.4621'
index_price: '0.4864755122500001'
mark_price: '0.4863'
settlement_price: '0.4864'
open_interest: '153090'
channel: 'RISK'
type: 'UPDATE'
time: '2024-11-07T00:00:59.950Z'
product_id: 'ENA-PERP'
}
type CoinbaseInternationalFundingMessage = {
sequence: 108521023
funding_rate: '0.000009'
is_final: false
channel: 'FUNDING'
type: 'UPDATE'
time: '2024-11-07T00:00:51.068Z'
product_id: 'DEGEN-PERP'
}
================================================
FILE: src/mappers/coinflex.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, DerivativeTicker, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://docs.coinflex.com/v2/#websocket-api-subscriptions-public
export const coinflexTradesMapper: Mapper<'coinflex', Trade> = {
canHandle(message: CoinflexTrades) {
return message.table === 'trade'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
}
]
},
*map(coinflexTrades: CoinflexTrades, localTimestamp: Date): IterableIterator {
for (const trade of coinflexTrades.data) {
yield {
type: 'trade',
symbol: trade.marketCode,
exchange: 'coinflex',
id: trade.tradeId,
price: Number(trade.price),
amount: Number(trade.quantity),
side: trade.side === 'SELL' ? 'sell' : 'buy',
timestamp: new Date(Number(trade.timestamp)),
localTimestamp: localTimestamp
}
}
}
}
const mapBookLevel = (level: CoinflexBookLevel) => {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
export const coinflexBookChangeMapper: Mapper<'coinflex', BookChange> = {
canHandle(message: CoinflexBookDepthMessage) {
return message.table === 'futures/depth'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'futures/depth',
symbols
}
]
},
*map(depthMessage: CoinflexBookDepthMessage, localTimestamp: Date): IterableIterator {
for (const change of depthMessage.data) {
yield {
type: 'book_change',
symbol: change.instrumentId,
exchange: 'coinflex',
isSnapshot: depthMessage.action === 'partial',
bids: change.bids.map(mapBookLevel),
asks: change.asks.map(mapBookLevel),
timestamp: new Date(Number(change.timestamp)),
localTimestamp
}
}
}
}
export class CoinflexDerivativeTickerMapper implements Mapper<'coinflex', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: CoinflexTickerMessage) {
return message.table === 'ticker'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'ticker',
symbols
} as const
]
}
*map(message: CoinflexTickerMessage, localTimestamp: Date): IterableIterator {
for (const ticker of message.data) {
// exclude spot symbols
if (ticker.marketCode.split('-').length === 2) {
continue
}
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(ticker.marketCode, 'coinflex')
if (ticker.markPrice !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(ticker.markPrice))
}
if (ticker.openInterest !== undefined) {
pendingTickerInfo.updateOpenInterest(Number(ticker.openInterest))
}
if (ticker.last !== undefined) {
pendingTickerInfo.updateLastPrice(Number(ticker.last))
}
pendingTickerInfo.updateTimestamp(new Date(Number(ticker.timestamp)))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
type CoinflexTrades = {
data: [
{
side: 'SELL' | 'BUY'
quantity: string
price: string
marketCode: string
tradeId: string
timestamp: string
}
]
table: 'trade'
}
type CoinflexBookLevel = [number | string, number | string]
type CoinflexBookDepthMessage = {
data: [
{
instrumentId: string
asks: CoinflexBookLevel[]
bids: CoinflexBookLevel[]
timestamp: string
}
]
action: 'partial'
table: 'futures/depth'
}
type CoinflexTickerMessage = {
data: [
{
last: string
markPrice?: string
marketCode: string
openInterest: string
timestamp: string
}
]
table: 'ticker'
}
================================================
FILE: src/mappers/cryptocom.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, Exchange, BookTicker, Trade, DerivativeTicker } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
export class CryptoComTradesMapper implements Mapper<'crypto-com', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: CryptoComTradeMessage) {
return message.result !== undefined && message.result.channel === 'trade'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
} as const
]
}
*map(message: CryptoComTradeMessage, localTimestamp: Date): IterableIterator {
message.result.data.reverse()
for (const item of message.result.data) {
const trade: Trade = {
type: 'trade',
symbol: message.result.instrument_name,
exchange: this._exchange,
id: item.d.toString(),
price: Number(item.p),
amount: Number(item.q),
side: item.s === 'BUY' ? 'buy' : 'sell',
timestamp: new Date(item.t),
localTimestamp
}
yield trade
}
}
}
export class CryptoComBookChangeMapper implements Mapper<'crypto-com', BookChange> {
constructor(protected readonly _exchange: Exchange) {}
canHandle(message: CryptoComBookMessage) {
return message.result !== undefined && message.result.channel.startsWith('book')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'book',
symbols
} as const
]
}
*map(message: CryptoComBookMessage, localTimestamp: Date) {
if (message.result.data === undefined || message.result.data[0] === undefined) {
return
}
const bids = (message.result.channel === 'book' ? message.result.data[0].bids : message.result.data[0].update.bids) || []
const asks = (message.result.channel === 'book' ? message.result.data[0].asks : message.result.data[0].update.asks) || []
yield {
type: 'book_change',
symbol: message.result.instrument_name,
exchange: this._exchange,
isSnapshot: message.result.channel === 'book',
bids: bids.map(this._mapBookLevel),
asks: asks.map(this._mapBookLevel),
timestamp: new Date(message.result.data[0].t),
localTimestamp
} as const
}
private _mapBookLevel(level: [number | string, number | string]) {
return { price: Number(level[0]), amount: Number(level[1]) }
}
}
export class CryptoComBookTickerMapper implements Mapper<'crypto-com', BookTicker> {
constructor(protected readonly _exchange: Exchange) {}
canHandle(message: CryptoComTickerMessage) {
return message.result !== undefined && message.result.channel === 'ticker'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'ticker',
symbols
} as const
]
}
*map(message: CryptoComTickerMessage, localTimestamp: Date) {
for (const item of message.result.data) {
const bookTicker: BookTicker = {
type: 'book_ticker',
symbol: message.result.instrument_name,
exchange: this._exchange,
askAmount: item.ks !== undefined && item.ks !== null ? Number(item.ks) : undefined,
askPrice: item.k !== undefined && item.k !== null ? Number(item.k) : undefined,
bidPrice: item.b !== undefined && item.b !== null ? Number(item.b) : undefined,
bidAmount: item.bs !== undefined && item.bs !== null ? Number(item.bs) : undefined,
timestamp: new Date(item.t),
localTimestamp: localTimestamp
}
yield bookTicker
}
}
}
export class CryptoComDerivativeTickerMapper implements Mapper<'crypto-com', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
private readonly _indexPrices = new Map()
constructor(protected readonly exchange: Exchange) {}
canHandle(message: CryptoComDerivativesTickerMessage | CryptoComIndexMessage | CryptoComMarkPriceMessage | CryptoComFundingMessage) {
if (message.result === undefined) {
return false
}
if (message.result.instrument_name === undefined) {
return false
}
// spot symbols
if (message.result.instrument_name.includes('_')) {
return false
}
// options
if (message.result.instrument_name.split('-').length === 3) {
return false
}
return (
message.result.channel === 'ticker' ||
message.result.channel === 'index' ||
message.result.channel === 'mark' ||
message.result.channel === 'funding'
)
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
let indexes: string[] = []
if (symbols !== undefined) {
indexes = [...new Set(symbols.map((s) => `${s.split('-')[0]}-INDEX`))]
}
const filters = [
{
channel: 'ticker',
symbols
} as const,
{
channel: 'index',
symbols: indexes
} as const,
{
channel: 'mark',
symbols
} as const,
{
channel: 'funding',
symbols
} as const,
{
channel: 'estimatedfunding',
symbols
} as const
]
return filters
}
*map(
message:
| CryptoComDerivativesTickerMessage
| CryptoComIndexMessage
| CryptoComMarkPriceMessage
| CryptoComFundingMessage
| CryptoComEstFundingMessage,
localTimestamp: Date
): IterableIterator {
if (message.result.channel === 'index') {
this._indexPrices.set(message.result.instrument_name.split('-')[0], Number(message.result.data[0].v))
return
}
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.result.instrument_name, this.exchange)
const lastIndexPrice = this._indexPrices.get(message.result.instrument_name.split('-')[0])
if (lastIndexPrice !== undefined) {
pendingTickerInfo.updateIndexPrice(lastIndexPrice)
}
if (message.result.channel === 'ticker') {
if (message.result.data[0].a !== null && message.result.data[0].a !== undefined) {
pendingTickerInfo.updateLastPrice(Number(message.result.data[0].a))
}
if (message.result.data[0].oi !== null && message.result.data[0].oi !== undefined) {
pendingTickerInfo.updateOpenInterest(Number(message.result.data[0].oi))
}
}
if (message.result.channel === 'mark') {
if (message.result.data[0].v !== null && message.result.data[0].v !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(message.result.data[0].v))
}
}
if (message.result.channel === 'funding') {
if (message.result.data[0].v !== null && message.result.data[0].v !== undefined) {
pendingTickerInfo.updateFundingRate(Number(message.result.data[0].v))
const nextFundingTimestamp = new Date(message.result.data[0].t)
nextFundingTimestamp.setUTCHours(nextFundingTimestamp.getUTCHours() + 1)
nextFundingTimestamp.setUTCMinutes(0, 0, 0)
pendingTickerInfo.updateFundingTimestamp(nextFundingTimestamp)
}
}
if (message.result.channel === 'estimatedfunding') {
if (message.result.data[0] && message.result.data[0].v !== null && message.result.data[0].v !== undefined) {
pendingTickerInfo.updatePredictedFundingRate(Number(message.result.data[0].v))
}
}
pendingTickerInfo.updateTimestamp(new Date(message.result.data[0].t))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
type CryptoComTradeMessage =
| {
method: 'subscribe'
result: {
instrument_name: 'ETH_CRO' // instrument_name
subscription: 'trade.ETH_CRO'
channel: 'trade'
data: [
{
p: 162.12 // price
q: 11.085 // quantity
s: 'BUY' // side
d: 1210447366 // trade id
t: 1587523078844 // trade time
dataTime: 0 // please ignore this field
}
]
}
}
| {
id: -1
code: 0
method: 'subscribe'
result: {
channel: 'trade'
subscription: 'trade.BTCUSD-PERP'
instrument_name: 'BTCUSD-PERP'
data: [{ d: '4611686018439397540'; t: 1653992578435; p: '31603.5'; q: '0.1000'; s: 'BUY'; i: 'BTCUSD-PERP' }]
}
}
type CryptoComBookMessage =
| {
code: 0
method: 'subscribe'
result: {
instrument_name: 'ETH_CRO'
subscription: 'book.ETH_CRO.150'
channel: 'book'
depth: 150
data: [
{
bids: [number, number][]
asks: [number, number][]
t: 1659311999933
s: 788293808
}
]
}
}
| {
code: 0
method: 'subscribe'
result: {
instrument_name: 'DOT_USDT'
subscription: 'book.DOT_USDT.150'
channel: 'book.update'
depth: 150
data: [
{
update: { bids: [number, number][]; asks: [number, number][] }
t: 1659312000046
s: 763793123
}
]
}
}
| {
id: -1
code: 0
method: 'subscribe'
result: {
channel: 'book.update'
subscription: 'book.BTCUSD-PERP.50'
instrument_name: 'BTCUSD-PERP'
depth: 50
data: [
{
update: { asks: [string, string][]; bids: [string, string][] }
t: 1653992578436
tt: 1653992578428
u: 72560693920
pu: 72560688000
cs: 380529173
}
]
}
}
type CryptoComTickerMessage =
| {
code: 0
method: 'subscribe'
result: {
instrument_name: 'GODS_USDT'
subscription: 'ticker.GODS_USDT'
channel: 'ticker'
data: [
{
i: 'GODS_USDT'
b: 0.4262
bs?: 0.1
k: 0.4272
ks?: 0.2
a: 0.4272
t: 1659311999946
v: 100623.01
vv: 42986.1541
h: 0.4624
l: 0.4229
c: -0.0062
pc: -1.4302
}
]
}
}
| CryptoComDerivativesTickerMessage
type CryptoComDerivativesTickerMessage =
| {
id: -1
code: 0
method: 'subscribe'
result: {
channel: 'ticker'
instrument_name: 'BTCUSD-PERP'
subscription: 'ticker.BTCUSD-PERP'
data: [
{
h: '32222.5'
l: '30240.0'
a: '31611.0'
c: '0.0320'
b: '31613.0'
bs?: '0.1000'
k: '31613.5'
ks?: '0.2000'
i: 'BTCUSD-PERP'
v: '13206.4884'
vv: '433945264.39'
oi: '318.5162'
t: 1653992543383
}
]
}
}
| {
id: 2
method: 'subscribe'
code: 0
result: {
instrument_name: 'ESUSD-PERP'
subscription: 'ticker.ESUSD-PERP'
channel: 'ticker'
data: [
{
h: '0.09625'
l: '0.09230'
a: '0.09481'
c: '-0.0038'
b: '0.09451'
bs: '1'
k: '0.09452'
ks: '8461'
i: 'ESUSD-PERP'
v: '115'
vv: '10.84'
oi: '78522'
t: 1765238404604
}
]
}
}
| {
id: -1
code: 0
method: 'subscribe'
result: {
channel: 'ticker'
instrument_name: 'MATIC_USD'
subscription: 'ticker.MATIC_USD'
id: 1
data: [
{
h: '1.24383'
l: '1.18086'
a: '1.19604'
c: '-0.0315'
b: '1.19591'
bs: '0.1'
k: '1.19643'
ks: '0.7'
i: 'MATIC_USD'
v: '854908.9'
vv: '1043976.96'
oi: '0'
t: 1677628802241
}
]
}
}
type CryptoComIndexMessage = {
id: -1
method: 'subscribe'
code: 0
result: {
instrument_name: 'BTCUSD-INDEX'
subscription: 'index.BTCUSD-INDEX'
channel: 'index'
data: [{ v: '31601.35'; t: 1653992545000 }]
}
}
type CryptoComMarkPriceMessage = {
id: 1
method: 'subscribe'
code: 0
result: {
instrument_name: 'BTCUSD-PERP'
subscription: 'mark.BTCUSD-PERP'
channel: 'mark'
data: [{ v: '31606.3'; t: 1653992543000 }]
}
}
type CryptoComFundingMessage = {
id: -1
method: 'subscribe'
code: 0
result: {
instrument_name: 'BTCUSD-PERP'
subscription: 'funding.BTCUSD-PERP'
channel: 'funding'
data: [{ v: '0.00000700'; t: 1653992579000 }]
}
}
type CryptoComEstFundingMessage = {
id: 1
method: 'subscribe'
code: 0
result: {
instrument_name: 'AAVEUSD-PERP'
subscription: 'estimatedfunding.AAVEUSD-PERP'
channel: 'estimatedfunding'
data: [{ v: '0.000039493'; t: 1727308799000 }]
}
}
================================================
FILE: src/mappers/cryptofacilities.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Liquidation, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://www.cryptofacilities.com/resources/hc/en-us/categories/115000132213-API
export const cryptofacilitiesTradesMapper: Mapper<'cryptofacilities', Trade> = {
canHandle(message: CryptofacilitiesTrade | CryptofacilitiesTicker | CryptofacilitiesBookSnapshot | CryptofacilitiesBookUpdate) {
return message.feed === 'trade' && message.event === undefined
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
}
]
},
*map(trade: CryptofacilitiesTrade, localTimestamp: Date): IterableIterator {
yield {
type: 'trade',
symbol: trade.product_id,
exchange: 'cryptofacilities',
id: trade.uid,
price: trade.price,
amount: trade.qty,
side: trade.side,
timestamp: new Date(trade.time),
localTimestamp: localTimestamp
}
}
}
const mapBookLevel = ({ price, qty }: CryptofacilitiesBookLevel) => {
return { price, amount: qty < 0 ? 0 : qty }
}
export const cryptofacilitiesBookChangeMapper: Mapper<'cryptofacilities', BookChange> = {
canHandle(message: CryptofacilitiesTrade | CryptofacilitiesTicker | CryptofacilitiesBookSnapshot | CryptofacilitiesBookUpdate) {
return message.event === undefined && (message.feed === 'book' || message.feed === 'book_snapshot')
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'book',
symbols
},
{
channel: 'book_snapshot',
symbols
}
]
},
*map(message: CryptofacilitiesBookSnapshot | CryptofacilitiesBookUpdate, localTimestamp: Date): IterableIterator {
if (message.feed === 'book_snapshot') {
yield {
type: 'book_change',
symbol: message.product_id,
exchange: 'cryptofacilities',
isSnapshot: true,
bids: message.bids.map(mapBookLevel),
asks: message.asks.map(mapBookLevel),
timestamp: message.timestamp !== undefined ? new Date(message.timestamp) : localTimestamp,
localTimestamp: localTimestamp
}
} else {
const isAsk = message.side === 'sell'
const update = [
{
price: message.price,
amount: message.qty < 0 ? 0 : message.qty
}
]
yield {
type: 'book_change',
symbol: message.product_id,
exchange: 'cryptofacilities',
isSnapshot: false,
bids: isAsk ? [] : update,
asks: isAsk ? update : [],
timestamp: message.timestamp !== undefined ? new Date(message.timestamp) : localTimestamp,
localTimestamp: localTimestamp
}
}
}
}
export class CryptofacilitiesDerivativeTickerMapper implements Mapper<'cryptofacilities', DerivativeTicker> {
constructor(private readonly _useRelativeFundingRate: boolean) {}
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: CryptofacilitiesTrade | CryptofacilitiesTicker | CryptofacilitiesBookSnapshot | CryptofacilitiesBookUpdate) {
return message.feed === 'ticker' && message.event === undefined
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'ticker',
symbols
} as const
]
}
*map(ticker: CryptofacilitiesTicker, localTimestamp: Date): IterableIterator {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(ticker.product_id, 'cryptofacilities')
if (ticker.next_funding_rate_time === 0) {
return
}
if (this._useRelativeFundingRate) {
pendingTickerInfo.updateFundingRate(ticker.relative_funding_rate)
pendingTickerInfo.updatePredictedFundingRate(ticker.relative_funding_rate_prediction)
} else {
pendingTickerInfo.updateFundingRate(ticker.funding_rate)
pendingTickerInfo.updatePredictedFundingRate(ticker.funding_rate_prediction)
}
pendingTickerInfo.updateFundingTimestamp(
ticker.next_funding_rate_time !== undefined ? new Date(ticker.next_funding_rate_time) : undefined
)
pendingTickerInfo.updateIndexPrice(ticker.index)
pendingTickerInfo.updateMarkPrice(ticker.markPrice)
pendingTickerInfo.updateOpenInterest(ticker.openInterest)
pendingTickerInfo.updateLastPrice(ticker.last)
pendingTickerInfo.updateTimestamp(new Date(ticker.time))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
export const cryptofacilitiesLiquidationsMapper: Mapper<'cryptofacilities', Liquidation> = {
canHandle(message: CryptofacilitiesTrade | CryptofacilitiesTicker | CryptofacilitiesBookSnapshot | CryptofacilitiesBookUpdate) {
return message.feed === 'trade' && message.event === undefined && message.type === 'liquidation'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
}
]
},
*map(liquidationTrade: CryptofacilitiesTrade, localTimestamp: Date): IterableIterator {
yield {
type: 'liquidation',
symbol: liquidationTrade.product_id,
exchange: 'cryptofacilities',
id: liquidationTrade.uid,
price: liquidationTrade.price,
amount: liquidationTrade.qty,
side: liquidationTrade.side,
timestamp: new Date(liquidationTrade.time),
localTimestamp: localTimestamp
}
}
}
export const cryptofacilitiesBookTickerMapper: Mapper<'cryptofacilities', BookTicker> = {
canHandle(message: CryptofacilitiesTicker) {
return message.feed === 'ticker' && message.event === undefined
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'ticker',
symbols
}
]
},
*map(cryptofacilitiesTicker: CryptofacilitiesTicker, localTimestamp: Date): IterableIterator {
const ticker: BookTicker = {
type: 'book_ticker',
symbol: cryptofacilitiesTicker.product_id,
exchange: 'cryptofacilities',
askAmount: cryptofacilitiesTicker.ask_size,
askPrice: cryptofacilitiesTicker.ask,
bidPrice: cryptofacilitiesTicker.bid,
bidAmount: cryptofacilitiesTicker.bid_size,
timestamp: new Date(cryptofacilitiesTicker.time),
localTimestamp: localTimestamp
}
yield ticker
}
}
type CryptofacilitiesTrade = {
feed: 'trade'
type: 'liquidation' | 'fill'
uid: string | undefined
event: undefined
product_id: string
side: 'buy' | 'sell'
time: number
qty: number
price: number
}
type CryptofacilitiesTicker =
| {
feed: 'ticker'
event: undefined
product_id: string
index: number
last: number
openInterest: number
markPrice: number
funding_rate: number | undefined
funding_rate_prediction: number | undefined
next_funding_rate_time: number | undefined
time: number
bid: number | undefined
ask: number | undefined
bid_size: number | undefined
ask_size: number | undefined
relative_funding_rate: undefined
relative_funding_rate_prediction: undefined
}
| {
time: 1680307200005
product_id: 'PF_1INCHUSD'
event: undefined
funding_rate: -1.861241614653e-6
funding_rate_prediction: -4.87669653882e-6
relative_funding_rate: number | undefined
relative_funding_rate_prediction: number | undefined
next_funding_rate_time: 1680307200000
feed: 'ticker'
bid: 0.5609
ask: 0.5621
bid_size: 1123.0
ask_size: 8931.0
volume: 10902.0
dtm: 0
leverage: '10x'
index: 0.56158
premium: -0.0
last: 0.5594
change: -1.0086710316758118
suspended: false
tag: 'perpetual'
pair: '1INCH:USD'
openInterest: 27481.0
markPrice: 0.56147544277
maturityTime: 0
post_only: false
volumeQuote: 6028.1795
}
type CryptofacilitiesBookLevel = {
price: number
qty: number
}
type CryptofacilitiesBookSnapshot = {
feed: 'book_snapshot'
event: undefined
product_id: string
timestamp: number | undefined
bids: CryptofacilitiesBookLevel[]
asks: CryptofacilitiesBookLevel[]
}
type CryptofacilitiesBookUpdate = {
feed: 'book'
event: undefined
product_id: string
side: 'buy' | 'sell'
price: number
qty: number
timestamp: number | undefined
}
================================================
FILE: src/mappers/delta.ts
================================================
import { fromMicroSecondsToDate, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
export class DeltaTradesMapper implements Mapper<'delta', Trade> {
constructor(private _useV2Channels: boolean) {}
canHandle(message: DeltaTrade) {
return message.type === (this._useV2Channels ? 'all_trades' : 'recent_trade')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: this._useV2Channels ? 'all_trades' : 'recent_trade',
symbols
} as const
]
}
*map(message: DeltaTrade, localTimestamp: Date): IterableIterator {
yield {
type: 'trade',
symbol: message.symbol,
exchange: 'delta',
id: undefined,
price: Number(message.price),
amount: Number(message.size),
side: message.buyer_role === 'taker' ? 'buy' : 'sell',
timestamp: fromMicroSecondsToDate(message.timestamp),
localTimestamp: localTimestamp
}
}
}
const mapBookLevel = (level: DeltaBookLevel) => {
return {
price: Number(level.limit_price),
amount: Number(level.size)
}
}
const mapL2Level = (level: DeltaL2Level) => {
return {
price: Number(level[0]),
amount: Number(level[1])
}
}
export class DeltaBookChangeMapper implements Mapper<'delta', BookChange> {
constructor(private readonly _useL2UpdatesChannel: boolean) {}
canHandle(message: DeltaL2OrderBook | DeltaL2UpdateMessage) {
if (this._useL2UpdatesChannel) {
return message.type === 'l2_updates'
}
return message.type === 'l2_orderbook'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
if (this._useL2UpdatesChannel) {
return [
{
channel: 'l2_updates',
symbols
} as const
]
}
return [
{
channel: 'l2_orderbook',
symbols
} as const
]
}
*map(message: DeltaL2OrderBook | DeltaL2UpdateMessage, localTimestamp: Date): IterableIterator {
if (message.type === 'l2_updates') {
yield {
type: 'book_change',
symbol: message.symbol,
exchange: 'delta',
isSnapshot: message.action === 'snapshot',
bids: message.bids !== undefined ? message.bids.map(mapL2Level) : [],
asks: message.asks !== undefined ? message.asks.map(mapL2Level) : [],
timestamp: message.timestamp !== undefined ? fromMicroSecondsToDate(message.timestamp) : localTimestamp,
localTimestamp
}
} else {
if (message.buy === undefined && message.sell === undefined) {
return
}
yield {
type: 'book_change',
symbol: message.symbol,
exchange: 'delta',
isSnapshot: true,
bids: message.buy !== undefined ? message.buy.map(mapBookLevel) : [],
asks: message.sell !== undefined ? message.sell.map(mapBookLevel) : [],
timestamp: message.timestamp !== undefined ? fromMicroSecondsToDate(message.timestamp) : localTimestamp,
localTimestamp
}
}
}
}
export class DeltaDerivativeTickerMapper implements Mapper<'delta', DerivativeTicker> {
constructor(private _useV2Channels: boolean) {}
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: DeltaTrade | DeltaMarkPrice | DeltaFundingRate) {
return (
message.type === (this._useV2Channels ? 'all_trades' : 'recent_trade') ||
message.type === 'funding_rate' ||
message.type === 'mark_price'
)
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: this._useV2Channels ? 'all_trades' : 'recent_trade',
symbols
} as const,
{
channel: 'funding_rate',
symbols
} as const,
{
channel: 'mark_price',
symbols
} as const
]
}
*map(message: DeltaTrade | DeltaMarkPrice | DeltaFundingRate, localTimestamp: Date): IterableIterator {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.symbol.replace('MARK:', ''), 'delta')
if (message.type === 'recent_trade' || message.type === 'all_trades') {
pendingTickerInfo.updateLastPrice(Number(message.price))
}
if (message.type === 'mark_price') {
pendingTickerInfo.updateMarkPrice(Number(message.price))
}
if (message.type === 'funding_rate') {
if (message.funding_rate !== undefined) {
pendingTickerInfo.updateFundingRate(Number(message.funding_rate))
}
if (message.predicted_funding_rate !== undefined) {
pendingTickerInfo.updatePredictedFundingRate(Number(message.predicted_funding_rate))
}
if (message.next_funding_realization !== undefined) {
pendingTickerInfo.updateFundingTimestamp(fromMicroSecondsToDate(message.next_funding_realization))
}
}
pendingTickerInfo.updateTimestamp(fromMicroSecondsToDate(message.timestamp))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
export class DeltaBookTickerMapper implements Mapper<'delta', BookTicker> {
canHandle(message: DeltaL1Message) {
return message.type === 'l1_orderbook'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'l1_orderbook',
symbols
} as const
]
}
*map(message: DeltaL1Message, localTimestamp: Date) {
const ticker: BookTicker = {
type: 'book_ticker',
symbol: message.symbol,
exchange: 'delta',
askAmount: message.ask_qty !== undefined ? Number(message.ask_qty) : undefined,
askPrice: message.best_ask !== undefined ? Number(message.best_ask) : undefined,
bidPrice: message.best_bid !== undefined ? Number(message.best_bid) : undefined,
bidAmount: message.bid_qty !== undefined ? Number(message.bid_qty) : undefined,
timestamp: message.timestamp !== undefined ? fromMicroSecondsToDate(message.timestamp) : localTimestamp,
localTimestamp: localTimestamp
}
yield ticker
}
}
type DeltaTrade = {
buyer_role: 'taker' | 'maker'
price: string
size: number | string
symbol: string
timestamp: number
type: 'recent_trade' | 'all_trades'
}
type DeltaBookLevel = {
limit_price: string
size: number | string
}
type DeltaL2OrderBook = {
buy: DeltaBookLevel[]
sell: DeltaBookLevel[]
symbol: string
timestamp?: number
type: 'l2_orderbook'
}
type DeltaMarkPrice = {
price: string
symbol: string
timestamp: number
type: 'mark_price'
}
type DeltaFundingRate = {
funding_rate?: string | number
next_funding_realization?: number
predicted_funding_rate?: number
symbol: string
timestamp: number
type: 'funding_rate'
}
type DeltaL2Level = [string, string]
type DeltaL2UpdateMessage =
| {
action: 'snapshot'
asks: DeltaL2Level[]
bids: DeltaL2Level[]
cs: 220729409
sequence_no: 3660223
symbol: string
timestamp: 1680307203021223
type: 'l2_updates'
}
| {
action: 'update'
asks: DeltaL2Level[]
bids: DeltaL2Level[]
cs: 2728204214
sequence_no: 3660224
symbol: string
timestamp: 1680307203771239
type: 'l2_updates'
}
type DeltaL1Message = {
ask_qty: '1950'
best_ask: '4964.5'
best_bid: '4802'
bid_qty: '4356'
last_sequence_no: 1680307203966299
last_updated_at: 1680307203784000
product_id: 103877
symbol: 'P-BTC-33000-210423'
timestamp: 1680307203966299
type: 'l1_orderbook'
}
================================================
FILE: src/mappers/deribit.ts
================================================
import { asNumberIfValid } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Liquidation, OptionSummary, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://docs.deribit.com/v2/#subscriptions
function deribitCasing(symbols?: string[]) {
if (symbols !== undefined) {
return symbols.map((symbol) => {
if (symbol.endsWith('-C') || symbol.endsWith('-P')) {
const parts = symbol.split('-')
if (parts[2] !== undefined && parts[2].toUpperCase().includes('D')) {
parts[2] = parts[2].replace('D', 'd')
return parts.join('-')
} else {
return symbol.toUpperCase()
}
} else {
return symbol.toUpperCase()
}
})
}
return
}
export const deribitTradesMapper: Mapper<'deribit', Trade> = {
canHandle(message: any) {
const channel = message.params !== undefined ? (message.params.channel as string | undefined) : undefined
if (channel === undefined) {
return false
}
return channel.startsWith('trades')
},
getFilters(symbols?: string[]) {
symbols = deribitCasing(symbols)
return [
{
channel: 'trades',
symbols
}
]
},
*map(message: DeribitTradesMessage, localTimestamp: Date): IterableIterator {
for (const deribitTrade of message.params.data) {
yield {
type: 'trade',
symbol: deribitTrade.instrument_name.toUpperCase(),
exchange: 'deribit',
id: deribitTrade.trade_id,
price: deribitTrade.price,
amount: deribitTrade.amount,
side: deribitTrade.direction,
timestamp: new Date(deribitTrade.timestamp),
localTimestamp: localTimestamp
}
}
}
}
const mapBookLevel = (level: DeribitBookLevel) => {
const price = level[1]
const amount = level[0] === 'delete' ? 0 : level[2]
return { price, amount }
}
export const deribitBookChangeMapper: Mapper<'deribit', BookChange> = {
canHandle(message: any) {
const channel = message.params && (message.params.channel as string | undefined)
if (channel === undefined) {
return false
}
return channel.startsWith('book')
},
getFilters(symbols?: string[]) {
symbols = deribitCasing(symbols)
return [
{
channel: 'book',
symbols
}
]
},
*map(message: DeribitBookMessage, localTimestamp: Date): IterableIterator {
const deribitBookChange = message.params.data
// snapshots do not have prev_change_id set
const isSnapshot =
(deribitBookChange.type !== undefined && deribitBookChange.type === 'snapshot') ||
deribitBookChange.prev_change_id === undefined ||
deribitBookChange.prev_change_id === 0
yield {
type: 'book_change',
symbol: deribitBookChange.instrument_name.toUpperCase(),
exchange: 'deribit',
isSnapshot,
bids: deribitBookChange.bids.map(mapBookLevel),
asks: deribitBookChange.asks.map(mapBookLevel),
timestamp: new Date(deribitBookChange.timestamp),
localTimestamp: localTimestamp
}
}
}
export class DeribitDerivativeTickerMapper implements Mapper<'deribit', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: any) {
const channel = message.params && (message.params.channel as string | undefined)
if (channel === undefined) {
return false
}
return channel.startsWith('ticker') && (message.params.data.greeks === undefined || message.params.data.combo_state === 'active')
}
getFilters(symbols?: string[]) {
symbols = deribitCasing(symbols)
return [
{
channel: 'ticker',
symbols
} as const
]
}
*map(message: DeribitTickerMessage, localTimestamp: Date): IterableIterator {
const deribitTicker = message.params.data
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(deribitTicker.instrument_name, 'deribit')
pendingTickerInfo.updateFundingRate(deribitTicker.current_funding)
pendingTickerInfo.updateIndexPrice(deribitTicker.index_price)
pendingTickerInfo.updateMarkPrice(deribitTicker.mark_price)
pendingTickerInfo.updateOpenInterest(deribitTicker.open_interest)
pendingTickerInfo.updateLastPrice(deribitTicker.last_price)
pendingTickerInfo.updateTimestamp(new Date(deribitTicker.timestamp))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
export class DeribitOptionSummaryMapper implements Mapper<'deribit', OptionSummary> {
getFilters(symbols?: string[]) {
symbols = deribitCasing(symbols)
return [
{
channel: 'ticker',
symbols
} as const
]
}
canHandle(message: any) {
const channel = message.params && message.params.channel
if (channel === undefined) {
return false
}
return (
channel.startsWith('ticker') &&
(message.params.data.instrument_name.endsWith('-P') || message.params.data.instrument_name.endsWith('-C'))
)
}
*map(message: DeribitOptionTickerMessage, localTimestamp: Date) {
//MATIC_USDC-9MAR24-1d02-C
const optionInfo = message.params.data
//e.g., BTC-8JUN20-8750-P
const symbolParts = optionInfo.instrument_name.split('-')
const isPut = symbolParts[3] === 'P'
let strikePriceString = symbolParts[2]
if (strikePriceString.includes('d')) {
strikePriceString = strikePriceString.replace('d', '.')
}
const strikePrice = Number(strikePriceString)
const expirationDate = new Date(symbolParts[1] + 'Z')
expirationDate.setUTCHours(8)
const optionSummary: OptionSummary = {
type: 'option_summary',
symbol: optionInfo.instrument_name.toUpperCase(),
exchange: 'deribit',
optionType: isPut ? 'put' : 'call',
strikePrice,
expirationDate,
bestBidPrice: asNumberIfValid(optionInfo.best_bid_price),
bestBidAmount: asNumberIfValid(optionInfo.best_bid_amount),
bestBidIV: asNumberIfValid(optionInfo.bid_iv),
bestAskPrice: asNumberIfValid(optionInfo.best_ask_price),
bestAskAmount: asNumberIfValid(optionInfo.best_ask_amount),
bestAskIV: asNumberIfValid(optionInfo.ask_iv),
lastPrice: asNumberIfValid(optionInfo.last_price),
openInterest: optionInfo.open_interest,
markPrice: optionInfo.mark_price,
markIV: optionInfo.mark_iv,
delta: optionInfo.greeks.delta,
gamma: optionInfo.greeks.gamma,
vega: optionInfo.greeks.vega,
theta: optionInfo.greeks.theta,
rho: optionInfo.greeks.rho,
underlyingPrice: optionInfo.underlying_price,
underlyingIndex: optionInfo.underlying_index,
timestamp: new Date(optionInfo.timestamp),
localTimestamp: localTimestamp
}
yield optionSummary
}
}
export const deribitLiquidationsMapper: Mapper<'deribit', Liquidation> = {
canHandle(message: any) {
const channel = message.params !== undefined ? (message.params.channel as string | undefined) : undefined
if (channel === undefined) {
return false
}
return channel.startsWith('trades')
},
getFilters(symbols?: string[]) {
symbols = deribitCasing(symbols)
return [
{
channel: 'trades',
symbols
}
]
},
*map(message: DeribitTradesMessage, localTimestamp: Date): IterableIterator {
for (const deribitTrade of message.params.data) {
if (deribitTrade.liquidation !== undefined) {
let side
// "T" when liquidity taker side was under liquidation
if (deribitTrade.liquidation === 'T') {
side = deribitTrade.direction
} else {
// "M" when maker (passive) side of trade was under liquidation
side = deribitTrade.direction === 'buy' ? ('sell' as const) : ('buy' as const)
}
yield {
type: 'liquidation',
symbol: deribitTrade.instrument_name.toUpperCase(),
exchange: 'deribit',
id: deribitTrade.trade_id,
price: deribitTrade.price,
amount: deribitTrade.amount,
side,
timestamp: new Date(deribitTrade.timestamp),
localTimestamp: localTimestamp
}
}
}
}
}
export const deribitBookTickerMapper: Mapper<'deribit', BookTicker> = {
canHandle(message: any) {
const channel = message.params !== undefined ? (message.params.channel as string | undefined) : undefined
if (channel === undefined) {
return false
}
return channel.startsWith('ticker')
},
getFilters(symbols?: string[]) {
symbols = deribitCasing(symbols)
return [
{
channel: 'ticker',
symbols
}
]
},
*map(message: DeribitTickerMessage, localTimestamp: Date): IterableIterator {
const deribitTicker = message.params.data
const ticker: BookTicker = {
type: 'book_ticker',
symbol: deribitTicker.instrument_name.toUpperCase(),
exchange: 'deribit',
askAmount: asNumberIfValid(deribitTicker.best_ask_amount),
askPrice: asNumberIfValid(deribitTicker.best_ask_price),
bidPrice: asNumberIfValid(deribitTicker.best_bid_price),
bidAmount: asNumberIfValid(deribitTicker.best_bid_amount),
timestamp: new Date(deribitTicker.timestamp),
localTimestamp: localTimestamp
}
yield ticker
}
}
type DeribitMessage = {
params: {
channel: string
}
}
type DeribitTradesMessage = DeribitMessage & {
params: {
data: {
trade_id: string
instrument_name: string
timestamp: number
direction: 'buy' | 'sell'
price: number
amount: number
trade_seq: number
liquidation?: 'M' | 'T' | 'MT'
}[]
}
}
type DeribitBookLevel = ['new' | 'change' | 'delete', number, number]
type DeribitBookMessage = DeribitMessage & {
params: {
data: {
timestamp: number
instrument_name: string
prev_change_id?: number
bids: DeribitBookLevel[]
asks: DeribitBookLevel[]
type?: 'snapshot' | 'change'
}
}
}
type DeribitTickerMessage = DeribitMessage & {
params: {
data: {
timestamp: number
open_interest: number
last_price: number | undefined
mark_price: number
instrument_name: string
index_price: number
current_funding?: number
funding_8h?: number
best_bid_price: number | undefined
best_bid_amount: number | undefined
best_ask_price: number | undefined
best_ask_amount: number | undefined
}
}
}
type DeribitOptionTickerMessage = DeribitTickerMessage & {
params: {
data: {
underlying_price: number
underlying_index: string
timestamp: number
open_interest: number
mark_price: number
mark_iv: number
last_price: number | null
greeks: { vega: number; theta: number; rho: number; gamma: number; delta: number }
bid_iv: number | undefined
ask_iv: number | undefined
}
}
}
================================================
FILE: src/mappers/dydx.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, BookPriceLevel, DerivativeTicker, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
export class DydxTradesMapper implements Mapper<'dydx', Trade> {
canHandle(message: DyDxTrade) {
return message.channel === 'v3_trades' && message.type === 'channel_data'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'v3_trades',
symbols
} as const
]
}
*map(message: DyDxTrade, localTimestamp: Date): IterableIterator {
for (let trade of message.contents.trades) {
yield {
type: 'trade',
symbol: message.id,
exchange: 'dydx',
id: undefined,
price: Number(trade.price),
amount: Number(trade.size),
side: trade.side === 'SELL' ? 'sell' : 'buy',
timestamp: trade.createdAt ? new Date(trade.createdAt) : localTimestamp,
localTimestamp: localTimestamp
}
}
}
}
export class DydxBookChangeMapper implements Mapper<'dydx', BookChange> {
private _bidsOffsets: { [key: string]: { [key: string]: number | undefined } } = {}
private _asksOffsets: { [key: string]: { [key: string]: number | undefined } } = {}
canHandle(message: DyDxOrderbookSnapshot | DyDxOrderBookUpdate) {
return message.channel === 'v3_orderbook'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'v3_orderbook',
symbols
} as const
]
}
*map(message: DyDxOrderbookSnapshot | DyDxOrderBookUpdate, localTimestamp: Date): IterableIterator {
if (message.type === 'subscribed') {
this._bidsOffsets[message.id] = {}
this._asksOffsets[message.id] = {}
yield {
type: 'book_change',
symbol: message.id,
exchange: 'dydx',
isSnapshot: true,
bids: message.contents.bids.map((bid) => {
this._bidsOffsets[message.id][bid.price] = Number(bid.offset)
return {
price: Number(bid.price),
amount: Number(bid.size)
}
}),
asks: message.contents.asks.map((ask) => {
this._asksOffsets[message.id][ask.price] = Number(ask.offset)
return {
price: Number(ask.price),
amount: Number(ask.size)
}
}),
timestamp: localTimestamp,
localTimestamp
}
} else {
if (!message.contents) {
return
}
// https://docs.dydx.exchange/#orderbook
const updateOffset = Number(message.contents.offset)
const bookChange: BookChange = {
type: 'book_change',
symbol: message.id,
exchange: 'dydx',
isSnapshot: false,
bids: message.contents.bids
.map((bid) => {
const lastPriceLevelOffset = this._bidsOffsets[message.id] && this._bidsOffsets[message.id][bid[0]]
if (lastPriceLevelOffset !== undefined && lastPriceLevelOffset >= updateOffset) {
return
}
return {
price: Number(bid[0]),
amount: Number(bid[1])
}
})
.filter((b) => b !== undefined) as BookPriceLevel[],
asks: message.contents.asks
.map((ask) => {
const lastPriceLevelOffset = this._asksOffsets[message.id] && this._asksOffsets[message.id][ask[0]]
if (lastPriceLevelOffset !== undefined && lastPriceLevelOffset >= updateOffset) {
return
}
return {
price: Number(ask[0]),
amount: Number(ask[1])
}
})
.filter((b) => b !== undefined) as BookPriceLevel[],
timestamp: localTimestamp,
localTimestamp
}
if (!this._bidsOffsets[message.id]) {
this._bidsOffsets[message.id] = {}
}
for (const bid of message.contents.bids) {
this._bidsOffsets[message.id][bid[0]] = updateOffset
}
if (!this._asksOffsets[message.id]) {
this._asksOffsets[message.id] = {}
}
for (const ask of message.contents.asks) {
this._asksOffsets[message.id][ask[0]] = updateOffset
}
if (bookChange.bids.length > 0 || bookChange.asks.length > 0) {
yield bookChange
}
}
}
}
export class DydxDerivativeTickerMapper implements Mapper<'dydx', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: DydxMarketsSnapshot | DyDxMarketsUpdate | DyDxTrade) {
return message.channel === 'v3_markets' || (message.channel === 'v3_trades' && message.type === 'channel_data')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'v3_markets',
symbols: [] as string[]
} as const,
{
channel: 'v3_trades',
symbols
} as const
]
}
*map(message: DydxMarketsSnapshot | DyDxMarketsUpdate | DyDxTrade, localTimestamp: Date): IterableIterator {
if (message.channel === 'v3_trades') {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.id, 'dydx')
pendingTickerInfo.updateLastPrice(Number(message.contents.trades[message.contents.trades.length - 1].price))
return
}
const contents = message.type === 'subscribed' ? message.contents.markets : message.contents
for (const key in contents) {
const marketInfo = contents[key]
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(key, 'dydx')
if (marketInfo.indexPrice !== undefined) {
pendingTickerInfo.updateIndexPrice(Number(marketInfo.indexPrice))
}
if (marketInfo.oraclePrice !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(marketInfo.oraclePrice))
}
if (marketInfo.openInterest !== undefined) {
pendingTickerInfo.updateOpenInterest(Number(marketInfo.openInterest))
}
if (marketInfo.nextFundingRate !== undefined) {
pendingTickerInfo.updateFundingRate(Number(marketInfo.nextFundingRate))
}
if (marketInfo.nextFundingAt !== undefined) {
pendingTickerInfo.updateFundingTimestamp(new Date(marketInfo.nextFundingAt))
}
pendingTickerInfo.updateTimestamp(localTimestamp)
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
type DyDxTrade = {
type: 'channel_data'
connection_id: 'e368fe1e-a007-44bd-9532-8eacc81a8bbc'
message_id: 229
id: 'BTC-USD'
channel: 'v3_trades'
contents: {
trades: [{ size: '0.075'; side: 'SELL'; price: '57696'; createdAt: '2021-05-01T00:00:34.046Z' | undefined }]
}
}
type DyDxOrderbookSnapshot = {
type: 'subscribed'
connection_id: '22be6448-1464-45ff-ae7d-1204eac64d0f'
message_id: 1
channel: 'v3_orderbook'
id: '1INCH-USD'
contents: {
bids: [{ price: '5'; offset: '118546101'; size: '50' }]
asks: [{ price: '7'; offset: '120842096'; size: '20' }]
}
}
type DyDxOrderBookUpdate = {
type: 'channel_data'
connection_id: '22be6448-1464-45ff-ae7d-1204eac64d0f'
message_id: 161
id: '1INCH-USD'
channel: 'v3_orderbook'
contents: {
offset: '125090042'
bids: [string, string][]
asks: [string, string][]
}
}
type DydxMarketsSnapshot = {
type: 'subscribed'
connection_id: '8c11ee31-dbca-49fa-9df0-fc973948b7b5'
message_id: 3
channel: 'v3_markets'
contents: {
markets: {
[key: string]: {
market: 'BTC-USD'
status: 'ONLINE'
baseAsset: 'BTC'
quoteAsset: 'USD'
stepSize: '0.0001'
tickSize: '1'
indexPrice: '57794.7000'
oraclePrice: '57880.5200'
priceChange24H: '4257.9'
nextFundingRate: '0.0000587260'
nextFundingAt: '2021-05-01T00:00:00.000Z'
minOrderSize: '0.001'
type: 'PERPETUAL'
initialMarginFraction: '0.04'
maintenanceMarginFraction: '0.03'
volume24H: '4710467.697100'
trades24H: '663'
openInterest: '101.2026'
incrementalInitialMarginFraction: '0.01'
incrementalPositionSize: '0.5'
maxPositionSize: '30'
baselinePositionSize: '1.0'
allTimeLiquidationQuoteVolume: '3001153.615633'
dailyLiquidationQuoteVolume: '6047.074828'
}
}
}
}
type DyDxMarketsUpdate = {
type: 'channel_data'
connection_id: '8c11ee31-dbca-49fa-9df0-fc973948b7b5'
message_id: 221
channel: 'v3_markets'
contents: {
[key: string]: {
market: 'BTC-USD'
status: 'ONLINE'
baseAsset: 'BTC'
quoteAsset: 'USD'
stepSize: '0.0001'
tickSize: '1'
indexPrice: '57794.7000'
oraclePrice: '57880.5200'
priceChange24H: '4257.9'
nextFundingRate: '0.0000587260'
nextFundingAt: '2021-05-01T00:00:00.000Z'
minOrderSize: '0.001'
type: 'PERPETUAL'
initialMarginFraction: '0.04'
maintenanceMarginFraction: '0.03'
volume24H: '4710467.697100'
trades24H: '663'
openInterest: '101.2026'
incrementalInitialMarginFraction: '0.01'
incrementalPositionSize: '0.5'
maxPositionSize: '30'
baselinePositionSize: '1.0'
allTimeLiquidationQuoteVolume: '3001153.615633'
dailyLiquidationQuoteVolume: '6047.074828'
}
}
}
================================================
FILE: src/mappers/dydxv4.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, DerivativeTicker, Liquidation, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
export class DydxV4TradesMapper implements Mapper<'dydx-v4', Trade> {
canHandle(message: DyDxTrade) {
return message.channel === 'v4_trades' && message.type === 'channel_data'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'v4_trades',
symbols
} as const
]
}
*map(message: DyDxTrade, localTimestamp: Date): IterableIterator {
for (let trade of message.contents.trades) {
yield {
type: 'trade',
symbol: message.id,
exchange: 'dydx-v4',
id: trade.id,
price: Number(trade.price),
amount: Number(trade.size),
side: trade.side === 'SELL' ? 'sell' : 'buy',
timestamp: trade.createdAt ? new Date(trade.createdAt) : localTimestamp,
localTimestamp: localTimestamp
}
}
}
}
function mapSnapshotPriceLevel(level: { price: string; size: string }) {
return {
price: Number(level.price),
amount: Number(level.size)
}
}
function mapUpdatePriceLevel(level: [string, string]) {
return {
price: Number(level[0]),
amount: Number(level[1])
}
}
export class DydxV4BookChangeMapper implements Mapper<'dydx-v4', BookChange> {
canHandle(message: DyDxOrderbookSnapshot | DyDxOrderBookUpdate) {
return message.channel === 'v4_orderbook'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'v4_orderbook',
symbols
} as const
]
}
*map(message: DyDxOrderbookSnapshot | DyDxOrderBookUpdate, localTimestamp: Date): IterableIterator {
if (message.type === 'subscribed') {
yield {
type: 'book_change',
symbol: message.id,
exchange: 'dydx-v4',
isSnapshot: true,
bids: message.contents.bids.map(mapSnapshotPriceLevel),
asks: message.contents.asks.map(mapSnapshotPriceLevel),
timestamp: localTimestamp,
localTimestamp
}
} else {
if (!message.contents) {
return
}
const bookChange: BookChange = {
type: 'book_change',
symbol: message.id,
exchange: 'dydx-v4',
isSnapshot: false,
bids: message.contents.bids !== undefined ? message.contents.bids.map(mapUpdatePriceLevel) : [],
asks: message.contents.asks !== undefined ? message.contents.asks.map(mapUpdatePriceLevel) : [],
timestamp: localTimestamp,
localTimestamp
}
if (bookChange.bids.length > 0 || bookChange.asks.length > 0) {
yield bookChange
}
}
}
}
export class DydxV4DerivativeTickerMapper implements Mapper<'dydx-v4', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: DydxMarketsSnapshot | DyDxMarketsUpdate | DyDxTrade) {
return message.channel === 'v4_markets' || (message.channel === 'v4_trades' && message.type === 'channel_data')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'v4_markets',
symbols: [] as string[]
} as const,
{
channel: 'v4_trades',
symbols
} as const
]
}
*map(message: DydxMarketsSnapshot | DyDxMarketsUpdate | DyDxTrade, localTimestamp: Date): IterableIterator {
if (message.channel === 'v4_trades') {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.id, 'dydx-v4')
pendingTickerInfo.updateLastPrice(Number(message.contents.trades[message.contents.trades.length - 1].price))
return
}
if (message.type === 'subscribed' || (message.type === 'channel_data' && message.contents.trading !== undefined)) {
const contents = message.type === 'subscribed' ? message.contents.markets : message.contents.trading
for (const key in contents) {
const marketInfo = (contents as any)[key] as DydxMarketsSnapshotContent | DydxMarketTradeUpdate
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(key, 'dydx-v4')
if (marketInfo.oraclePrice !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(marketInfo.oraclePrice))
}
if (marketInfo.openInterest !== undefined) {
pendingTickerInfo.updateOpenInterest(Number(marketInfo.openInterest))
}
if (marketInfo.nextFundingRate !== undefined) {
pendingTickerInfo.updateFundingRate(Number(marketInfo.nextFundingRate))
}
pendingTickerInfo.updateTimestamp(localTimestamp)
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
if (message.type === 'channel_data' && message.contents.oraclePrices !== undefined) {
for (const key in message.contents.oraclePrices) {
const oraclePriceInfo = (message.contents.oraclePrices as any)[key] as OraclePriceInfo
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(key, 'dydx-v4')
if (oraclePriceInfo.oraclePrice !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(oraclePriceInfo.oraclePrice))
}
pendingTickerInfo.updateTimestamp(localTimestamp)
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
}
export class DydxV4LiquidationsMapper implements Mapper<'dydx-v4', Liquidation> {
canHandle(message: DyDxTrade) {
return message.channel === 'v4_trades' && message.type === 'channel_data'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'v4_trades',
symbols
} as const
]
}
*map(message: DyDxTrade, localTimestamp: Date): IterableIterator {
for (let trade of message.contents.trades) {
if (trade.type === 'LIQUIDATED') {
yield {
type: 'liquidation',
symbol: message.id,
exchange: 'dydx-v4',
id: trade.id,
price: Number(trade.price),
amount: Number(trade.size),
side: trade.side === 'SELL' ? 'sell' : 'buy',
timestamp: trade.createdAt ? new Date(trade.createdAt) : localTimestamp,
localTimestamp: localTimestamp
}
}
}
}
}
type DyDxTrade = {
type: 'channel_data'
connection_id: '3a2e4c0c-7579-4bf6-a570-e0979418bbe9'
message_id: 15897
id: 'BTC-USD'
channel: 'v4_trades'
version: '2.1.0'
contents: {
trades: [
{
id: '0165e6170000000200000002'
size: '0.0001'
price: '60392'
side: 'BUY' | 'SELL'
createdAt: '2024-08-23T00:00:57.627Z'
type: 'LIMIT' | 'LIQUIDATED'
}
]
}
}
type DyDxOrderbookSnapshot = {
type: 'subscribed'
connection_id: '67838890-75de-4bf3-a638-d7bcdea5f245'
message_id: 7
channel: 'v4_orderbook'
id: 'GRT-USD'
contents: {
bids: [{ price: '0.1547'; size: '35520' }]
asks: [{ price: '0.155'; size: '3220' }]
}
}
type DyDxOrderBookUpdate = {
type: 'channel_data'
connection_id: '00908030-4a70-43aa-9263-8ccdf57b5d40'
message_id: 10290
id: 'EOS-USD'
channel: 'v4_orderbook'
version: '1.0.0'
contents: { bids: [['0.1003', '2017130']]; asks: undefined | [['0.1003', '2017130']] }
}
type DydxMarketsSnapshot = {
type: 'subscribed'
connection_id: '3a2e4c0c-7579-4bf6-a570-e0979418bbe9'
message_id: 17
channel: 'v4_markets'
contents: {
markets: {
[key: string]: DydxMarketsSnapshotContent
}
}
}
type DydxMarketsSnapshotContent = {
clobPairId: '0'
ticker: 'BTC-USD'
status: 'ACTIVE'
oraclePrice: '60387.51779'
priceChange24H: '-782.58326'
volume24H: '247515340.0835'
trades24H: 73556
nextFundingRate: '0.00001351666666666667'
initialMarginFraction: '0.05'
maintenanceMarginFraction: '0.03'
openInterest: '648.2389'
atomicResolution: -10
quantumConversionExponent: -9
tickSize: '1'
stepSize: '0.0001'
stepBaseQuantums: 1000000
subticksPerTick: 100000
marketType: 'CROSS'
openInterestLowerCap: '0'
openInterestUpperCap: '0'
baseOpenInterest: '648.4278'
}
type DyDxMarketsUpdate =
| {
type: 'channel_data'
connection_id: '3a2e4c0c-7579-4bf6-a570-e0979418bbe9'
message_id: 15871
channel: 'v4_markets'
version: '1.0.0'
contents: {
oraclePrices: undefined
trading: {
'ETH-USD': DydxMarketTradeUpdate
}
}
}
| {
type: 'channel_data'
connection_id: '3a2e4c0c-7579-4bf6-a570-e0979418bbe9'
message_id: 50
channel: 'v4_markets'
version: '1.0.0'
contents: {
trading: undefined
oraclePrices: {
'ZERO-USD': OraclePriceInfo
}
}
}
type OraclePriceInfo = { oraclePrice: string; effectiveAt: string; effectiveAtHeight: string; marketId: number }
type DydxMarketTradeUpdate = {
id?: string
clobPairId?: string
ticker?: string
marketId?: number
oraclePrice: undefined
baseAsset?: string
quoteAsset?: string
initialMarginFraction?: string
maintenanceMarginFraction?: string
basePositionSize?: string
incrementalPositionSize?: string
maxPositionSize?: string
openInterest?: string
quantumConversionExponent?: number
atomicResolution?: number
subticksPerTick?: number
stepBaseQuantums?: number
priceChange24H?: string
volume24H?: string
trades24H?: number
nextFundingRate?: string
}
================================================
FILE: src/mappers/ftx.ts
================================================
import { asNumberIfValid, parseμs, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Exchange, Liquidation, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://docs.ftx.com/#websocket-api
export class FTXTradesMapper implements Mapper<'ftx' | 'ftx-us', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: FtxTrades | FtxOrderBook) {
if (message.data == undefined) {
return false
}
return message.channel === 'trades'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
} as const
]
}
*map(ftxTrades: FtxTrades, localTimestamp: Date): IterableIterator {
for (const ftxTrade of ftxTrades.data) {
const timestamp = new Date(ftxTrade.time)
timestamp.μs = parseμs(ftxTrade.time)
yield {
type: 'trade',
symbol: ftxTrades.market,
exchange: this._exchange,
id: ftxTrade.id !== null ? String(ftxTrade.id) : undefined,
price: ftxTrade.price,
amount: ftxTrade.size,
side: ftxTrade.side,
timestamp,
localTimestamp
}
}
}
}
export const mapBookLevel = (level: FtxBookLevel) => {
const price = level[0]
const amount = level[1]
return { price, amount }
}
export class FTXBookChangeMapper implements Mapper<'ftx' | 'ftx-us', BookChange> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: FtxTrades | FtxOrderBook) {
if (message.data == undefined) {
return false
}
return message.channel === 'orderbook'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'orderbook',
symbols
} as const
]
}
*map(ftxOrderBook: FtxOrderBook, localTimestamp: Date): IterableIterator {
const isEmptyUpdate = ftxOrderBook.type === 'update' && ftxOrderBook.data.bids.length === 0 && ftxOrderBook.data.asks.length === 0
if (isEmptyUpdate) {
return
}
const timestamp = new Date(ftxOrderBook.data.time * 1000)
timestamp.μs = Math.floor(ftxOrderBook.data.time * 1000000) % 1000
yield {
type: 'book_change',
symbol: ftxOrderBook.market,
exchange: this._exchange,
isSnapshot: ftxOrderBook.type === 'partial',
bids: ftxOrderBook.data.bids.map(mapBookLevel),
asks: ftxOrderBook.data.asks.map(mapBookLevel),
timestamp,
localTimestamp
}
}
}
export class FTXDerivativeTickerMapper implements Mapper<'ftx', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
constructor(private readonly _exchange: Exchange) {}
canHandle(message: FTXInstrument) {
if (message.data == undefined) {
return false
}
return message.channel === 'instrument'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'instrument',
symbols: symbols !== undefined ? symbols.filter((s) => s.includes('/') === false) : undefined
} as const
]
}
*map(message: FTXInstrument, localTimestamp: Date): IterableIterator {
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.market, this._exchange)
const { stats, info } = message.data
const currentFundingTimestamp = pendingTickerInfo.getCurrentFundingTimestamp()
const updatedFundingTimestamp = stats.nextFundingTime !== undefined ? new Date(stats.nextFundingTime) : undefined
// due to how instrument info messages are sourced (from REST API) it can sometimes return data that is stale (cached perhaps by the API)
// let's skip such messages
const isStaleInfo =
updatedFundingTimestamp !== undefined &&
currentFundingTimestamp !== undefined &&
currentFundingTimestamp.valueOf() > updatedFundingTimestamp.valueOf()
if (isStaleInfo) {
return
}
if (updatedFundingTimestamp !== undefined) {
pendingTickerInfo.updateFundingTimestamp(updatedFundingTimestamp)
pendingTickerInfo.updateFundingRate(stats.nextFundingRate)
}
pendingTickerInfo.updateIndexPrice(info.index)
pendingTickerInfo.updateMarkPrice(info.mark)
pendingTickerInfo.updateLastPrice(info.last)
pendingTickerInfo.updateOpenInterest(stats.openInterest)
pendingTickerInfo.updateTimestamp(localTimestamp)
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
export class FTXLiquidationsMapper implements Mapper<'ftx', Liquidation> {
canHandle(message: FtxTrades | FtxOrderBook) {
if (message.data == undefined) {
return false
}
return message.channel === 'trades'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
} as const
]
}
*map(ftxTrades: FtxTrades, localTimestamp: Date): IterableIterator {
for (const ftxTrade of ftxTrades.data) {
if (ftxTrade.liquidation) {
const timestamp = new Date(ftxTrade.time)
timestamp.μs = parseμs(ftxTrade.time)
yield {
type: 'liquidation',
symbol: ftxTrades.market,
exchange: 'ftx',
id: ftxTrade.id !== null ? String(ftxTrade.id) : undefined,
price: ftxTrade.price,
amount: ftxTrade.size,
side: ftxTrade.side,
timestamp,
localTimestamp
}
}
}
}
}
export class FTXBookTickerMapper implements Mapper<'ftx' | 'ftx-us', BookTicker> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: FTXTicker) {
if (message.data == undefined) {
return false
}
return message.channel === 'ticker'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'ticker',
symbols
} as const
]
}
*map(ftxTicker: FTXTicker, localTimestamp: Date): IterableIterator {
const timestamp = new Date(ftxTicker.data.time * 1000)
timestamp.μs = Math.floor(ftxTicker.data.time * 1000000) % 1000
const ticker: BookTicker = {
type: 'book_ticker',
symbol: ftxTicker.market,
exchange: this._exchange,
askAmount: asNumberIfValid(ftxTicker.data.askSize),
askPrice: asNumberIfValid(ftxTicker.data.ask),
bidPrice: asNumberIfValid(ftxTicker.data.bid),
bidAmount: asNumberIfValid(ftxTicker.data.bidSize),
timestamp,
localTimestamp: localTimestamp
}
yield ticker
}
}
type FtxTrades = {
channel: 'trades'
market: string
type: 'update'
data: {
id: number | null
price: number
size: number
side: 'buy' | 'sell'
time: string
liquidation?: boolean
}[]
}
type FtxBookLevel = [number, number]
type FtxOrderBook = {
channel: 'orderbook'
market: string
type: 'update' | 'partial'
data: { time: number; bids: FtxBookLevel[]; asks: FtxBookLevel[] }
}
type FTXInstrument = {
channel: 'instrument'
market: string
type: 'update'
data: {
stats: {
nextFundingRate?: number
nextFundingTime?: string
openInterest: number
}
info: {
last: number
mark: number
index: number
}
}
}
type FTXTicker = {
channel: 'ticker'
market: string
type: 'update'
data: { bid: number; ask: number; bidSize: number; askSize: number; last: number; time: number }
}
================================================
FILE: src/mappers/gateio.ts
================================================
import { debug } from '../debug.ts'
import { CircularBuffer, fromMicroSecondsToDate, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, Exchange, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
export class GateIOV4OrderBookV2ChangeMapper implements Mapper<'gate-io', BookChange> {
constructor(protected readonly exchange: Exchange) {}
canHandle(message: GateV4OrderBookV2Message) {
return message.channel === 'spot.obu' && message.event === 'update'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'obu',
symbols
} as const
]
}
*map(message: GateV4OrderBookV2Message, localTimestamp: Date) {
const result = message.result
const symbol = this.extractSymbolFromStream(result.s)
const isSnapshot = result.full === true
const bookChange: BookChange = {
type: 'book_change',
symbol,
exchange: this.exchange,
isSnapshot,
bids: (result.b || []).map(this.mapBookLevel),
asks: (result.a || []).map(this.mapBookLevel),
timestamp: new Date(result.t),
localTimestamp
}
yield bookChange
}
protected mapBookLevel(level: [string, string]) {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
private extractSymbolFromStream(streamName: string): string {
const lastDotIndex = streamName.lastIndexOf('.')
return streamName.slice(3, lastDotIndex)
}
}
//v4
export class GateIOV4BookChangeMapper implements Mapper<'gate-io', BookChange> {
protected readonly symbolToDepthInfoMapping: {
[key: string]: LocalDepthInfo
} = {}
constructor(protected readonly exchange: Exchange, protected readonly ignoreBookSnapshotOverlapError: boolean) {}
canHandle(message: GateV4OrderBookUpdate | Gatev4OrderBookSnapshot) {
if (message.channel === undefined) {
return false
}
if (message.event !== 'update' && message.event !== 'snapshot') {
return false
}
return message.channel.endsWith('order_book_update')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'order_book_update',
symbols
} as const
]
}
*map(message: GateV4OrderBookUpdate | Gatev4OrderBookSnapshot, localTimestamp: Date) {
const symbol = message.event === 'snapshot' ? message.symbol : message.result.s
if (this.symbolToDepthInfoMapping[symbol] === undefined) {
this.symbolToDepthInfoMapping[symbol] = {
bufferedUpdates: new CircularBuffer(2000)
}
}
const symbolDepthInfo = this.symbolToDepthInfoMapping[symbol]
const snapshotAlreadyProcessed = symbolDepthInfo.snapshotProcessed
// first check if received message is snapshot and process it as such if it is
if (message.event === 'snapshot') {
// if we've already received 'manual' snapshot, ignore if there is another one
if (snapshotAlreadyProcessed) {
return
}
// produce snapshot book_change
const snapshotData = message.result
// mark given symbol depth info that has snapshot processed
symbolDepthInfo.lastUpdateId = snapshotData.id
symbolDepthInfo.snapshotProcessed = true
// if there were any depth updates buffered, let's proccess those by adding to or updating the initial snapshot
for (const update of symbolDepthInfo.bufferedUpdates.items()) {
const bookChange = this.mapBookDepthUpdate(update, localTimestamp)
if (bookChange !== undefined) {
for (const bid of update.b) {
const matchingBid = snapshotData.bids.find((b) => b[0] === bid[0])
if (matchingBid !== undefined) {
matchingBid[1] = bid[1]
} else {
snapshotData.bids.push(bid)
}
}
for (const ask of update.a) {
const matchingAsk = snapshotData.asks.find((a) => a[0] === ask[0])
if (matchingAsk !== undefined) {
matchingAsk[1] = ask[1]
} else {
snapshotData.asks.push(ask)
}
}
}
}
// remove all buffered updates
symbolDepthInfo.bufferedUpdates.clear()
const bookChange: BookChange = {
type: 'book_change',
symbol,
exchange: this.exchange,
isSnapshot: true,
bids: snapshotData.bids.map(this.mapBookLevel),
asks: snapshotData.asks.map(this.mapBookLevel),
timestamp: new Date(snapshotData.update),
localTimestamp
}
yield bookChange
} else if (snapshotAlreadyProcessed) {
// snapshot was already processed let's map the message as normal book_change
const bookChange = this.mapBookDepthUpdate(message.result as DepthData, localTimestamp)
if (bookChange !== undefined) {
yield bookChange
}
} else {
const depthUpdate = message.result as DepthData
symbolDepthInfo.bufferedUpdates.append(depthUpdate)
}
}
protected mapBookDepthUpdate(depthUpdateData: DepthData, localTimestamp: Date): BookChange | undefined {
// we can safely assume here that depthContext and lastUpdateId aren't null here as this is method only works
// when we've already processed the snapshot
const depthContext = this.symbolToDepthInfoMapping[depthUpdateData.s]!
const lastUpdateId = depthContext.lastUpdateId!
// Drop any event where u is <= lastUpdateId in the snapshot
if (depthUpdateData.u <= lastUpdateId) {
return
}
// The first processed event should have U <= lastUpdateId+1 AND u >= lastUpdateId+1.
if (!depthContext.validatedFirstUpdate) {
// if there is new instrument added it can have empty book at first and that's normal
const bookSnapshotIsEmpty = lastUpdateId == -1
if ((depthUpdateData.U <= lastUpdateId + 1 && depthUpdateData.u >= lastUpdateId + 1) || bookSnapshotIsEmpty) {
depthContext.validatedFirstUpdate = true
} else {
const message = `Book depth snapshot has no overlap with first update, update ${JSON.stringify(
depthUpdateData
)}, lastUpdateId: ${lastUpdateId}, exchange ${this.exchange}`
if (this.ignoreBookSnapshotOverlapError) {
depthContext.validatedFirstUpdate = true
debug(message)
} else {
throw new Error(message)
}
}
}
return {
type: 'book_change',
symbol: depthUpdateData.s,
exchange: this.exchange,
isSnapshot: false,
bids: depthUpdateData.b.map(this.mapBookLevel),
asks: depthUpdateData.a.map(this.mapBookLevel),
timestamp: fromMicroSecondsToDate(depthUpdateData.t),
localTimestamp: localTimestamp
}
}
protected mapBookLevel(level: [string, string]) {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
}
export class GateIOV4BookTickerMapper implements Mapper<'gate-io', BookTicker> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: GateV4BookTicker) {
if (message.channel === undefined) {
return false
}
if (message.event !== 'update') {
return false
}
return message.channel.endsWith('book_ticker')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'book_ticker',
symbols
} as const
]
}
*map(bookTickerResponse: GateV4BookTicker, localTimestamp: Date) {
const gateBookTicker = bookTickerResponse.result
const ticker: BookTicker = {
type: 'book_ticker',
symbol: gateBookTicker.s,
exchange: this._exchange,
askAmount: gateBookTicker.A !== undefined ? Number(gateBookTicker.A) : undefined,
askPrice: gateBookTicker.a !== undefined ? Number(gateBookTicker.a) : undefined,
bidPrice: gateBookTicker.b !== undefined ? Number(gateBookTicker.b) : undefined,
bidAmount: gateBookTicker.B !== undefined ? Number(gateBookTicker.B) : undefined,
timestamp: gateBookTicker.t !== undefined ? new Date(gateBookTicker.t) : localTimestamp,
localTimestamp: localTimestamp
}
yield ticker
}
}
export class GateIOV4TradesMapper implements Mapper<'gate-io', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: GateV4Trade) {
if (message.channel === undefined) {
return false
}
if (message.event !== 'update') {
return false
}
return message.channel.endsWith('trades')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
} as const
]
}
*map(tradesMessage: GateV4Trade, localTimestamp: Date): IterableIterator {
yield {
type: 'trade',
symbol: tradesMessage.result.currency_pair,
exchange: this._exchange,
id: tradesMessage.result.id.toString(),
price: Number(tradesMessage.result.price),
amount: Number(tradesMessage.result.amount),
side: tradesMessage.result.side == 'sell' ? 'sell' : 'buy',
timestamp: new Date(Number(tradesMessage.result.create_time_ms)),
localTimestamp: localTimestamp
}
}
}
// v3 https://www.gate.io/docs/websocket/index.html
export class GateIOTradesMapper implements Mapper<'gate-io', Trade> {
private readonly _seenSymbols = new Set()
constructor(private readonly _exchange: Exchange) {}
canHandle(message: any) {
return message.method === 'trades.update'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
} as const
]
}
*map(tradesMessage: GateIOTrades, localTimestamp: Date): IterableIterator {
const symbol = tradesMessage.params[0]
if (!tradesMessage.params[1]) {
return
}
// gate io sends trades from newest to oldest for some reason
for (const gateIOTrade of tradesMessage.params[1].reverse()) {
// always ignore first returned trade as it's a 'stale' trade, which has already been published before disconnect
if (this._seenSymbols.has(symbol) === false) {
this._seenSymbols.add(symbol)
break
}
const timestamp = new Date(gateIOTrade.time * 1000)
timestamp.μs = Math.floor(gateIOTrade.time * 1000000) % 1000
yield {
type: 'trade',
symbol,
exchange: this._exchange,
id: gateIOTrade.id.toString(),
price: Number(gateIOTrade.price),
amount: Number(gateIOTrade.amount),
side: gateIOTrade.type == 'sell' ? 'sell' : 'buy',
timestamp,
localTimestamp: localTimestamp
}
}
}
}
const mapBookLevel = (level: GateIODepthLevel) => {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
export class GateIOBookChangeMapper implements Mapper<'gate-io', BookChange> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: any) {
return message.method === 'depth.update'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'depth',
symbols
} as const
]
}
*map(depthMessage: GateIODepth, localTimestamp: Date): IterableIterator {
const symbol = depthMessage.params[2]
const isSnapshot = depthMessage.params[0]
const bids = Array.isArray(depthMessage.params[1].bids) ? depthMessage.params[1].bids : []
const asks = Array.isArray(depthMessage.params[1].asks) ? depthMessage.params[1].asks : []
const timestamp = depthMessage.params[1].current !== undefined ? new Date(depthMessage.params[1].current * 1000) : localTimestamp
yield {
type: 'book_change',
symbol,
exchange: this._exchange,
isSnapshot,
bids: bids.map(mapBookLevel),
asks: asks.map(mapBookLevel),
timestamp: timestamp,
localTimestamp: localTimestamp
}
}
}
type GateIOTrade = {
id: number
time: number
price: string
amount: string
type: 'sell' | 'buy'
}
type GateIOTrades = {
method: 'trades.update'
params: [string, GateIOTrade[]]
}
type GateIODepthLevel = [string, string]
type GateIODepth = {
method: 'depth.update'
params: [
boolean,
{
bids?: GateIODepthLevel[]
asks?: GateIODepthLevel[]
current: 1669860180.632
update: 1669860180.632
},
string
]
}
type GateV4Trade = {
time: 1682689046
time_ms: 1682689046133
channel: 'spot.trades'
event: 'update'
result: {
id: 5541729596
create_time: 1682689046
create_time_ms: '1682689046123.0'
side: 'sell'
currency_pair: 'SUSD_USDT'
amount: '8.5234'
price: '0.9782'
}
}
type GateV4BookTicker = {
time: 1682689046
time_ms: 1682689046142
channel: 'spot.book_ticker'
event: 'update'
result: { t: 1682689046131; u: 517377894; s: 'ETC_ETH'; b: '0.010326'; B: '0.001'; a: '0.010366'; A: '10' }
}
type Gatev4OrderBookSnapshot = {
channel: 'spot.order_book_update'
event: 'snapshot'
generated: true
symbol: '1ART_USDT'
result: {
id: 154857784
current: 1682689045318
update: 1682689045056
asks: [string, string][]
bids: [string, string][]
}
}
type GateV4OrderBookUpdate = {
time: 1682689045
time_ms: 1682689045532
channel: 'spot.order_book_update'
event: 'update'
result: {
lastUpdateId: undefined
t: 1682689045424
e: 'depthUpdate'
E: 1682689045
s: '1ART_USDT'
U: 154857785
u: 154857785
b: [string, string][]
a: [string, string][]
}
}
type LocalDepthInfo = {
bufferedUpdates: CircularBuffer
snapshotProcessed?: boolean
lastUpdateId?: number
validatedFirstUpdate?: boolean
}
type DepthData = {
lastUpdateId: undefined
t: number
s: string
U: number
u: number
b: [string, string][]
a: [string, string][]
}
type GateV4OrderBookV2Message = {
channel: 'spot.obu'
event: 'update'
time_ms: number
result: {
t: number
s: string
u: number
U?: number
full?: boolean
b?: [string, string][]
a?: [string, string][]
}
}
================================================
FILE: src/mappers/gateiofutures.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, DerivativeTicker, Exchange, Trade, BookTicker } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://www.gate.io/docs/futures/ws/index.html
export class GateIOFuturesTradesMapper implements Mapper<'gate-io-futures', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: any) {
return message.channel === 'futures.trades' && message.event === 'update'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trades',
symbols
} as const
]
}
*map(tradesMessage: GateIOFuturesTrades, localTimestamp: Date): IterableIterator {
for (const trade of tradesMessage.result) {
const timestamp = trade.create_time_ms !== undefined ? new Date(trade.create_time_ms) : new Date(trade.create_time * 1000)
const size = Number(trade.size)
yield {
type: 'trade',
symbol: trade.contract,
exchange: this._exchange,
id: trade.id.toString(),
price: Number(trade.price),
amount: Math.abs(size),
side: size < 0 ? 'sell' : 'buy',
timestamp,
localTimestamp: localTimestamp
}
}
}
}
const mapBookLevel = (level: GateIOFuturesSnapshotLevel) => {
const price = Number(level.p)
const size = Number(level.s)
return { price, amount: Math.abs(size) }
}
export class GateIOFuturesBookChangeMapper implements Mapper<'gate-io-futures', BookChange> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: GateIOFuturesOrderBookSnapshot | GateIOFuturesOrderBookUpdate) {
return message.channel === 'futures.order_book' && (message.event === 'all' || message.event === 'update')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'order_book',
symbols
} as const
]
}
*map(depthMessage: GateIOFuturesOrderBookSnapshot | GateIOFuturesOrderBookUpdate, localTimestamp: Date): IterableIterator {
if (depthMessage.event === 'all') {
if (depthMessage.result.t === 0) {
return
}
const timestamp =
depthMessage.time_ms !== undefined
? new Date(depthMessage.time_ms)
: depthMessage.result.t !== undefined
? new Date(depthMessage.result.t)
: new Date(depthMessage.time * 1000)
// snapshot
yield {
type: 'book_change',
symbol: depthMessage.result.contract,
exchange: this._exchange,
isSnapshot: true,
bids: depthMessage.result.bids.map(mapBookLevel),
asks: depthMessage.result.asks.map(mapBookLevel),
timestamp,
localTimestamp: localTimestamp
}
} else if (depthMessage.result.length > 0) {
// update
const timestamp = depthMessage.result[0].t !== undefined ? new Date(depthMessage.result[0].t) : new Date(depthMessage.time * 1000)
yield {
type: 'book_change',
symbol: depthMessage.result[0].c,
exchange: this._exchange,
isSnapshot: false,
bids: depthMessage.result.filter((l) => Number(l.s) >= 0).map(mapBookLevel),
asks: depthMessage.result.filter((l) => Number(l.s) <= 0).map(mapBookLevel),
timestamp,
localTimestamp: localTimestamp
}
}
}
}
export class GateIOFuturesDerivativeTickerMapper implements Mapper<'gate-io-futures', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: GateIOFuturesTicker) {
return message.channel === 'futures.tickers' && message.event === 'update'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'tickers',
symbols
} as const
]
}
*map(message: GateIOFuturesTicker, localTimestamp: Date): IterableIterator {
const tickers = Array.isArray(message.result) ? message.result : [message.result]
for (const futuresTicker of tickers) {
if (futuresTicker.contract === undefined) {
return
}
const timestamp = message.time_ms !== undefined ? new Date(message.time_ms) : new Date(message.time * 1000)
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(futuresTicker.contract, 'gate-io-futures')
pendingTickerInfo.updateFundingRate(Number(futuresTicker.funding_rate))
pendingTickerInfo.updatePredictedFundingRate(Number(futuresTicker.funding_rate_indicative))
pendingTickerInfo.updateIndexPrice(Number(futuresTicker.index_price))
pendingTickerInfo.updateMarkPrice(Number(futuresTicker.mark_price))
pendingTickerInfo.updateLastPrice(Number(futuresTicker.last))
pendingTickerInfo.updateTimestamp(timestamp)
if (futuresTicker.total_size !== undefined) {
pendingTickerInfo.updateOpenInterest(Number(futuresTicker.total_size))
}
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
export class GateIOFuturesBookTickerMapper implements Mapper<'gate-io-futures', BookTicker> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: any) {
return message.channel === 'futures.book_ticker' && message.event === 'update'
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'book_ticker',
symbols
} as const
]
}
*map(gateIoFuturesBookTickerMessage: GateIOFuturesBookTicker, localTimestamp: Date): IterableIterator {
const gateIoFuturesBookTicker = gateIoFuturesBookTickerMessage.result
const askAmount = Number(gateIoFuturesBookTicker.A)
const bidAmount = Number(gateIoFuturesBookTicker.B)
if (gateIoFuturesBookTicker.t === 0) {
return
}
const ticker: BookTicker = {
type: 'book_ticker',
symbol: gateIoFuturesBookTicker.s,
exchange: this._exchange,
askAmount: askAmount !== 0 ? askAmount : undefined,
askPrice: gateIoFuturesBookTicker.a !== '' ? Number(gateIoFuturesBookTicker.a) : undefined,
bidPrice: gateIoFuturesBookTicker.b !== '' ? Number(gateIoFuturesBookTicker.b) : undefined,
bidAmount: bidAmount !== 0 ? bidAmount : undefined,
timestamp: gateIoFuturesBookTicker.t !== undefined ? new Date(gateIoFuturesBookTicker.t) : localTimestamp,
localTimestamp: localTimestamp
}
yield ticker
}
}
type GateIOFuturesTrade = {
size: number | string
id: number
create_time: number
create_time_ms?: number
price: string
contract: string
}
type GateIOFuturesTrades = {
time: number
channel: 'futures.trades'
event: 'update'
result: GateIOFuturesTrade[]
}
type GateIOFuturesSnapshotLevel = { p: string; s: number | string }
type GateIOFuturesOrderBookSnapshot = {
time: number
channel: 'futures.order_book'
event: 'all'
time_ms: number | undefined
result: {
t?: number
contract: string
asks: GateIOFuturesSnapshotLevel[]
bids: GateIOFuturesSnapshotLevel[]
}
}
type GateIOFuturesOrderBookUpdate = {
time: number
channel: 'futures.order_book'
event: 'update'
result: {
t?: number
p: string
s: number | string
c: string
}[]
}
type GateIOFuturesTicker = {
time: number
time_ms?: number
channel: 'futures.tickers'
event: 'update'
result:
| [
{
contract: string
last: string
funding_rate: string
mark_price: string
index_price: string
funding_rate_indicative: string
total_size: string | undefined
}
]
| {
contract: string
last: string
funding_rate: string
mark_price: string
index_price: string
funding_rate_indicative: string
total_size: string | undefined
}
}
type GateIOFuturesBookTicker = {
id: null
time: 1648771200
channel: 'futures.book_ticker'
event: 'update'
error: null
result: { t: number; u: 3502782378; s: 'BTC_USD'; b: string; B: number | string; a: string; A: number | string }
}
================================================
FILE: src/mappers/gemini.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
// https://docs.gemini.com/websocket-api/#market-data-version-2
export const geminiTradesMapper: Mapper<'gemini', Trade> = {
canHandle(message: GeminiL2Updates | GeminiTrade) {
return message.type === 'trade'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
}
]
},
*map(geminiTrade: GeminiTrade, localTimestamp: Date): IterableIterator {
yield {
type: 'trade',
symbol: geminiTrade.symbol,
exchange: 'gemini',
id: String(geminiTrade.event_id),
price: Number(geminiTrade.price),
amount: Number(geminiTrade.quantity),
side: geminiTrade.side === 'buy' ? 'buy' : geminiTrade.side === 'sell' ? 'sell' : 'unknown',
timestamp: new Date(geminiTrade.timestamp),
localTimestamp: localTimestamp
}
}
}
const mapBookLevel = (level: GeminiBookLevel) => {
const price = Number(level[1])
const amount = Number(level[2])
return { price, amount }
}
export const geminiBookChangeMapper: Mapper<'gemini', BookChange> = {
canHandle(message: GeminiL2Updates | GeminiTrade) {
return message.type === 'l2_updates'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'l2_updates',
symbols
}
]
},
*map(geminiL2Updates: GeminiL2Updates, localTimestamp: Date): IterableIterator {
yield {
type: 'book_change',
symbol: geminiL2Updates.symbol,
exchange: 'gemini',
isSnapshot: geminiL2Updates.auction_events !== undefined,
bids: geminiL2Updates.changes.filter((c) => c[0] === 'buy').map(mapBookLevel),
asks: geminiL2Updates.changes.filter((c) => c[0] === 'sell').map(mapBookLevel),
timestamp: localTimestamp,
localTimestamp
}
}
}
type GeminiBookLevel = ['buy' | 'sell', string, string]
type GeminiL2Updates = {
type: 'l2_updates'
symbol: string
changes: GeminiBookLevel[]
auction_events: any[]
}
type GeminiTrade = {
type: 'trade'
symbol: string
event_id: number
timestamp: number
price: string
quantity: string
side: 'sell' | 'buy'
}
================================================
FILE: src/mappers/hitbtc.ts
================================================
import { upperCaseSymbols } from '../handy.ts'
import { BookChange, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
// https://api.hitbtc.com/#socket-market-data
export const hitBtcTradesMapper: Mapper<'hitbtc', Trade> = {
canHandle(message: HitBtcTradesMessage) {
return message.method !== undefined && message.method === 'updateTrades'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'updateTrades',
symbols
}
]
},
*map(message: HitBtcTradesMessage, localTimestamp: Date): IterableIterator {
for (const trade of message.params.data)
yield {
type: 'trade',
symbol: message.params.symbol,
exchange: 'hitbtc',
id: String(trade.id),
price: Number(trade.price),
amount: Number(trade.quantity),
side: trade.side,
timestamp: new Date(trade.timestamp),
localTimestamp: localTimestamp
}
}
}
const mapBookLevel = (level: HitBtcBookLevel) => {
const price = Number(level.price)
const amount = Number(level.size)
return { price, amount }
}
export const hitBtcBookChangeMapper: Mapper<'hitbtc', BookChange> = {
canHandle(message: HitBtcBookMessage) {
if (message.method === undefined) {
return false
}
return message.method === 'snapshotOrderbook' || message.method === 'updateOrderbook'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'snapshotOrderbook',
symbols
},
{
channel: 'updateOrderbook',
symbols
}
]
},
*map(message: HitBtcBookMessage, localTimestamp: Date): IterableIterator {
yield {
type: 'book_change',
symbol: message.params.symbol,
exchange: 'hitbtc',
isSnapshot: message.method === 'snapshotOrderbook',
bids: message.params.bid.map(mapBookLevel),
asks: message.params.ask.map(mapBookLevel),
timestamp: new Date(message.params.timestamp),
localTimestamp
}
}
}
type HitBtcMessage = {
method?: string
}
type HitBtcTradesMessage = HitBtcMessage & {
method: 'updateTrades'
params: {
data: {
id: number
price: string
quantity: string
side: 'buy' | 'sell'
timestamp: string
}[]
symbol: string
}
}
type HitBtcBookLevel = {
price: string
size: string
}
type HitBtcBookMessage = HitBtcMessage & {
method: 'snapshotOrderbook' | 'updateOrderbook'
params: {
ask: HitBtcBookLevel[]
bid: HitBtcBookLevel[]
symbol: string
timestamp: string
}
}
================================================
FILE: src/mappers/huobi.ts
================================================
import { asNumberIfValid, CircularBuffer, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Exchange, FilterForExchange, Liquidation, OptionSummary, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
// https://huobiapi.github.io/docs/spot/v1/en/#websocket-market-data
// https://github.com/huobiapi/API_Docs_en/wiki/WS_api_reference_en
export class HuobiTradesMapper
implements Mapper<'huobi' | 'huobi-dm' | 'huobi-dm-swap' | 'huobi-dm-linear-swap' | 'huobi-dm-options', Trade>
{
constructor(private readonly _exchange: Exchange) {}
canHandle(message: HuobiDataMessage) {
if (message.ch === undefined) {
return false
}
return message.ch.endsWith('.trade.detail')
}
getFilters(symbols?: string[]) {
symbols = normalizeSymbols(symbols)
return [
{
channel: 'trade',
symbols
} as const
]
}
*map(message: HuobiTradeDataMessage, localTimestamp: Date): IterableIterator {
const symbol = message.ch.split('.')[1].toUpperCase()
for (const huobiTrade of message.tick.data) {
yield {
type: 'trade',
symbol,
exchange: this._exchange,
id: String(huobiTrade.tradeId !== undefined ? huobiTrade.tradeId : huobiTrade.id),
price: huobiTrade.price,
amount: huobiTrade.amount,
side: huobiTrade.direction === 'buy' ? 'buy' : huobiTrade.direction === 'sell' ? 'sell' : 'unknown',
timestamp: new Date(huobiTrade.ts),
localTimestamp: localTimestamp
}
}
}
}
export class HuobiBookChangeMapper
implements Mapper<'huobi' | 'huobi-dm' | 'huobi-dm-swap' | 'huobi-dm-linear-swap' | 'huobi-dm-options', BookChange>
{
constructor(protected readonly _exchange: Exchange) {}
canHandle(message: HuobiDataMessage) {
if (message.ch === undefined) {
return false
}
return message.ch.includes('.depth.')
}
getFilters(symbols?: string[]) {
symbols = normalizeSymbols(symbols)
return [
{
channel: 'depth',
symbols
} as const
]
}
*map(message: HuobiDepthDataMessage, localTimestamp: Date) {
const symbol = message.ch.split('.')[1].toUpperCase()
const isSnapshot = 'event' in message.tick ? message.tick.event === 'snapshot' : 'update' in message ? false : true
const data = message.tick
const bids = Array.isArray(data.bids) ? data.bids : []
const asks = Array.isArray(data.asks) ? data.asks : []
if (bids.length === 0 && asks.length === 0) {
return
}
yield {
type: 'book_change',
symbol,
exchange: this._exchange,
isSnapshot,
bids: bids.map(this._mapBookLevel),
asks: asks.map(this._mapBookLevel),
timestamp: new Date(message.ts),
localTimestamp: localTimestamp
} as const
}
private _mapBookLevel(level: HuobiBookLevel) {
return { price: level[0], amount: level[1] }
}
}
function isSnapshot(message: HuobiMBPDataMessage | HuobiMBPSnapshot): message is HuobiMBPSnapshot {
return 'rep' in message
}
export class HuobiMBPBookChangeMapper implements Mapper<'huobi', BookChange> {
protected readonly symbolToMBPInfoMapping: {
[key: string]: MBPInfo
} = {}
constructor(protected readonly _exchange: Exchange) {}
canHandle(message: any) {
const channel = message.ch || message.rep
if (channel === undefined) {
return false
}
return channel.includes('.mbp.')
}
getFilters(symbols?: string[]) {
symbols = normalizeSymbols(symbols)
return [
{
channel: 'mbp',
symbols
} as const
]
}
*map(message: HuobiMBPDataMessage | HuobiMBPSnapshot, localTimestamp: Date) {
const symbol = (isSnapshot(message) ? message.rep : message.ch).split('.')[1].toUpperCase()
if (this.symbolToMBPInfoMapping[symbol] === undefined) {
this.symbolToMBPInfoMapping[symbol] = {
bufferedUpdates: new CircularBuffer(20)
}
}
const mbpInfo = this.symbolToMBPInfoMapping[symbol]
const snapshotAlreadyProcessed = mbpInfo.snapshotProcessed
if (isSnapshot(message)) {
if (message.data == null) {
return
}
const snapshotBids = message.data.bids.map(this._mapBookLevel)
const snapshotAsks = message.data.asks.map(this._mapBookLevel)
// if there were any depth updates buffered, let's proccess those by adding to or updating the initial snapshot
// when prevSeqNum >= snapshot seqNum
for (const update of mbpInfo.bufferedUpdates.items()) {
if (update.tick.prevSeqNum < message.data.seqNum) {
continue
}
const bookChange = this._mapMBPUpdate(update, symbol, localTimestamp)
if (bookChange !== undefined) {
for (const bid of bookChange.bids) {
const matchingBid = snapshotBids.find((b) => b.price === bid.price)
if (matchingBid !== undefined) {
matchingBid.amount = bid.amount
} else {
snapshotBids.push(bid)
}
}
for (const ask of bookChange.asks) {
const matchingAsk = snapshotAsks.find((a) => a.price === ask.price)
if (matchingAsk !== undefined) {
matchingAsk.amount = ask.amount
} else {
snapshotAsks.push(ask)
}
}
}
}
mbpInfo.snapshotProcessed = true
yield {
type: 'book_change',
symbol,
exchange: this._exchange,
isSnapshot: true,
bids: snapshotBids,
asks: snapshotAsks,
timestamp: new Date(message.ts),
localTimestamp
} as const
} else {
mbpInfo.bufferedUpdates.append(message)
if (snapshotAlreadyProcessed) {
// snapshot was already processed let's map the mbp message as normal book_change
const update = this._mapMBPUpdate(message, symbol, localTimestamp)
if (update !== undefined) {
yield update
}
}
}
}
private _mapMBPUpdate(message: HuobiMBPDataMessage, symbol: string, localTimestamp: Date) {
const bids = Array.isArray(message.tick.bids) ? message.tick.bids : []
const asks = Array.isArray(message.tick.asks) ? message.tick.asks : []
if (bids.length === 0 && asks.length === 0) {
return
}
return {
type: 'book_change',
symbol,
exchange: this._exchange,
isSnapshot: false,
bids: bids.map(this._mapBookLevel),
asks: asks.map(this._mapBookLevel),
timestamp: new Date(message.ts),
localTimestamp: localTimestamp
} as const
}
private _mapBookLevel(level: HuobiBookLevel) {
return { price: level[0], amount: level[1] }
}
}
function normalizeSymbols(symbols?: string[]) {
if (symbols !== undefined) {
return symbols.map((s) => {
// huobi-dm and huobi-dm-swap expect symbols to be upper cased
if (s.includes('_') || s.includes('-')) {
return s.toUpperCase()
}
// huobi global expects lower cased symbols
return s.toLowerCase()
})
}
return
}
export class HuobiDerivativeTickerMapper implements Mapper<'huobi-dm' | 'huobi-dm-swap' | 'huobi-dm-linear-swap', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
constructor(private readonly _exchange: Exchange) {}
canHandle(message: any) {
if (message.ch !== undefined) {
return message.ch.includes('.basis.') || message.ch.endsWith('.open_interest')
}
if (message.op === 'notify' && message.topic !== undefined) {
return message.topic.endsWith('.funding_rate')
}
return false
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
const filters: FilterForExchange['huobi-dm-swap'][] = [
{
channel: 'basis',
symbols
},
{
channel: 'open_interest',
symbols
}
]
if (this._exchange === 'huobi-dm-swap' || this._exchange === 'huobi-dm-linear-swap') {
filters.push({
channel: 'funding_rate',
symbols
})
}
return filters
}
*map(
message: HuobiBasisDataMessage | HuobiFundingRateNotification | HuobiOpenInterestDataMessage,
localTimestamp: Date
): IterableIterator {
if ('op' in message) {
// handle funding_rate notification message
const fundingInfo = message.data[0]
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(fundingInfo.contract_code, this._exchange)
pendingTickerInfo.updateFundingRate(Number(fundingInfo.funding_rate))
pendingTickerInfo.updateFundingTimestamp(new Date(Number(fundingInfo.settlement_time)))
pendingTickerInfo.updatePredictedFundingRate(Number(fundingInfo.estimated_rate))
pendingTickerInfo.updateTimestamp(new Date(message.ts))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
} else {
const symbol = message.ch.split('.')[1]
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(symbol, this._exchange)
// basis message
if ('tick' in message) {
pendingTickerInfo.updateIndexPrice(Number(message.tick.index_price))
pendingTickerInfo.updateLastPrice(Number(message.tick.contract_price))
} else {
// open interest message
const openInterest = message.data[0]
pendingTickerInfo.updateOpenInterest(Number(openInterest.volume))
}
pendingTickerInfo.updateTimestamp(new Date(message.ts))
if (pendingTickerInfo.hasChanged()) {
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
}
export class HuobiLiquidationsMapper implements Mapper<'huobi-dm' | 'huobi-dm-swap' | 'huobi-dm-linear-swap', Liquidation> {
private readonly _contractCodeToSymbolMap: Map = new Map()
private readonly _contractTypesSuffixes = { this_week: 'CW', next_week: 'NW', quarter: 'CQ', next_quarter: 'NQ' }
constructor(private readonly _exchange: Exchange) {}
canHandle(message: HuobiLiquidationOrder | HuobiContractInfo) {
if (message.op !== 'notify') {
return false
}
if (this._exchange === 'huobi-dm' && message.topic.endsWith('.contract_info')) {
this._updateContractCodeToSymbolMap(message as HuobiContractInfo)
}
return message.topic.endsWith('.liquidation_orders')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
if (this._exchange === 'huobi-dm') {
// huobi-dm for liquidations requires prividing different symbols which are indexes names for example 'BTC' or 'ETH'
// not futures names like 'BTC_NW'
// see https://huobiapi.github.io/docs/dm/v1/en/#subscribe-liquidation-order-data-no-authentication-sub
if (symbols !== undefined) {
symbols = symbols.map((s) => s.split('_')[0])
}
// we also need to subscribe to contract_info which will provide us information that will allow us to map
// liquidation message symbol and contract code to symbols we expect (BTC_NW etc)
return [
{
channel: 'liquidation_orders',
symbols
} as const,
{
channel: 'contract_info',
symbols
} as const
]
} else {
// huobi dm swap liquidations messages provide correct symbol & contract code
return [
{
channel: 'liquidation_orders',
symbols
} as const
]
}
}
private _updateContractCodeToSymbolMap(message: HuobiContractInfo) {
for (const item of message.data) {
this._contractCodeToSymbolMap.set(item.contract_code, `${item.symbol}_${this._contractTypesSuffixes[item.contract_type]}`)
}
}
*map(message: HuobiLiquidationOrder, localTimestamp: Date): IterableIterator {
for (const huobiLiquidation of message.data) {
let symbol = huobiLiquidation.contract_code
// huobi-dm returns index name as a symbol, not future alias, so we need to map it here
if (this._exchange === 'huobi-dm') {
const futureAliasSymbol = this._contractCodeToSymbolMap.get(huobiLiquidation.contract_code)
if (futureAliasSymbol === undefined) {
continue
}
symbol = futureAliasSymbol
}
yield {
type: 'liquidation',
symbol,
exchange: this._exchange,
id: undefined,
price: huobiLiquidation.price,
amount: huobiLiquidation.volume,
side: huobiLiquidation.direction === 'buy' ? 'buy' : huobiLiquidation.direction === 'sell' ? 'sell' : 'unknown',
timestamp: new Date(huobiLiquidation.created_at),
localTimestamp: localTimestamp
}
}
}
}
export class HuobiOptionsSummaryMapper implements Mapper<'huobi-dm-options', OptionSummary> {
private readonly _indexPrices = new Map()
private readonly _openInterest = new Map()
canHandle(message: HuobiOpenInterestDataMessage | HuobiOptionsIndexMessage | HuobiOptionsMarketIndexMessage) {
if (message.ch === undefined) {
return false
}
return message.ch.endsWith('.open_interest') || message.ch.endsWith('.option_index') || message.ch.endsWith('.option_market_index')
}
getFilters(symbols?: string[]) {
const indexes =
symbols !== undefined
? symbols.map((s) => {
const symbolParts = s.split('-')
return `${symbolParts[0]}-${symbolParts[1]}`
})
: undefined
return [
{
channel: `open_interest`,
symbols
} as const,
{
channel: `option_index`,
symbols: indexes
} as const,
{
channel: 'option_market_index',
symbols
} as const
]
}
*map(
message: HuobiOpenInterestDataMessage | HuobiOptionsIndexMessage | HuobiOptionsMarketIndexMessage,
localTimestamp: Date
): IterableIterator | undefined {
if (message.ch.endsWith('.option_index')) {
const indexUpdateMessage = message as HuobiOptionsIndexMessage
this._indexPrices.set(indexUpdateMessage.data.symbol, indexUpdateMessage.data.index_price)
return
}
if (message.ch.endsWith('.open_interest')) {
const openInterestMessage = message as HuobiOptionsOpenInterestMessage
for (const ioMessage of openInterestMessage.data) {
this._openInterest.set(ioMessage.contract_code, ioMessage.volume)
}
return
}
const marketIndexMessage = message as HuobiOptionsMarketIndexMessage
const symbolParts = marketIndexMessage.data.contract_code.split('-')
const expirationDate = new Date(`20${symbolParts[2].slice(0, 2)}-${symbolParts[2].slice(2, 4)}-${symbolParts[2].slice(4, 6)}Z`)
expirationDate.setUTCHours(8)
const underlying = `${symbolParts[0]}-${symbolParts[1]}`
const lastUnderlyingPrice = this._indexPrices.get(underlying)
const openInterest = this._openInterest.get(marketIndexMessage.data.contract_code)
const optionSummary: OptionSummary = {
type: 'option_summary',
symbol: marketIndexMessage.data.contract_code,
exchange: 'huobi-dm-options',
optionType: marketIndexMessage.data.option_right_type === 'P' ? 'put' : 'call',
strikePrice: Number(symbolParts[4]),
expirationDate,
bestBidPrice: asNumberIfValid(marketIndexMessage.data.bid_one),
bestBidAmount: undefined,
bestBidIV: asNumberIfValid(marketIndexMessage.data.iv_bid_one),
bestAskPrice: asNumberIfValid(marketIndexMessage.data.ask_one),
bestAskAmount: undefined,
bestAskIV: asNumberIfValid(marketIndexMessage.data.iv_ask_one),
lastPrice: asNumberIfValid(marketIndexMessage.data.last_price),
openInterest,
markPrice: marketIndexMessage.data.mark_price > 0 ? asNumberIfValid(marketIndexMessage.data.mark_price) : undefined,
markIV: asNumberIfValid(marketIndexMessage.data.iv_mark_price),
delta: asNumberIfValid(marketIndexMessage.data.delta),
gamma: asNumberIfValid(marketIndexMessage.data.gamma),
vega: asNumberIfValid(marketIndexMessage.data.vega),
theta: asNumberIfValid(marketIndexMessage.data.theta),
rho: undefined,
underlyingPrice: lastUnderlyingPrice,
underlyingIndex: underlying,
timestamp: new Date(marketIndexMessage.ts),
localTimestamp: localTimestamp
}
yield optionSummary
}
}
export class HuobiBookTickerMapper implements Mapper<'huobi' | 'huobi-dm' | 'huobi-dm-swap' | 'huobi-dm-linear-swap', BookTicker> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: HuobiDataMessage) {
if (message.ch === undefined) {
return false
}
return message.ch.endsWith('.bbo')
}
getFilters(symbols?: string[]) {
symbols = normalizeSymbols(symbols)
return [
{
channel: 'bbo',
symbols
} as const
]
}
*map(message: HuobiBBOMessage, localTimestamp: Date): IterableIterator {
const symbol = message.ch.split('.')[1].toUpperCase()
if ('quoteTime' in message.tick) {
if (message.tick.quoteTime === 0) {
return
}
yield {
type: 'book_ticker',
symbol,
exchange: this._exchange,
askAmount: asNumberIfValid(message.tick.askSize),
askPrice: asNumberIfValid(message.tick.ask),
bidPrice: asNumberIfValid(message.tick.bid),
bidAmount: asNumberIfValid(message.tick.bidSize),
timestamp: new Date(message.tick.quoteTime),
localTimestamp: localTimestamp
}
} else {
yield {
type: 'book_ticker',
symbol,
exchange: this._exchange,
askAmount: message.tick.ask !== undefined && message.tick.ask !== null ? asNumberIfValid(message.tick.ask[1]) : undefined,
askPrice: message.tick.ask !== undefined && message.tick.ask !== null ? asNumberIfValid(message.tick.ask[0]) : undefined,
bidPrice: message.tick.bid !== undefined && message.tick.bid !== null ? asNumberIfValid(message.tick.bid[0]) : undefined,
bidAmount: message.tick.bid !== undefined && message.tick.bid !== null ? asNumberIfValid(message.tick.bid[1]) : undefined,
timestamp: new Date(message.tick.ts),
localTimestamp: localTimestamp
}
}
}
}
type HuobiDataMessage = {
ch: string
}
type HuobiTradeDataMessage = HuobiDataMessage & {
tick: {
data: {
id: number
tradeId?: number
price: number
amount: number
direction: 'buy' | 'sell'
ts: number
}[]
}
}
type HuobiBookLevel = [number, number]
type HuobiDepthDataMessage = HuobiDataMessage &
(
| {
update?: boolean
ts: number
tick: {
bids: HuobiBookLevel[] | null
asks: HuobiBookLevel[] | null
}
}
| {
ts: number
tick: {
bids?: HuobiBookLevel[] | null
asks?: HuobiBookLevel[] | null
event: 'snapshot' | 'update'
}
}
)
type HuobiBasisDataMessage = HuobiDataMessage & {
ts: number
tick: {
index_price: string
contract_price: string
}
}
type HuobiFundingRateNotification = {
op: 'notify'
topic: string
ts: number
data: {
settlement_time: string
funding_rate: string
estimated_rate: string
contract_code: string
}[]
}
type HuobiOpenInterestDataMessage = HuobiDataMessage & {
ts: number
data: {
volume: number
}[]
}
type HuobiMBPDataMessage = HuobiDataMessage & {
ts: number
tick: {
bids?: HuobiBookLevel[] | null
asks?: HuobiBookLevel[] | null
seqNum: number
prevSeqNum: number
}
}
type HuobiMBPSnapshot = {
ts: number
rep: string
data?: {
bids: HuobiBookLevel[]
asks: HuobiBookLevel[]
seqNum: number
}
}
type MBPInfo = {
bufferedUpdates: CircularBuffer
snapshotProcessed?: boolean
}
type HuobiLiquidationOrder = {
op: 'notify'
topic: string
ts: number
data: {
symbol: string
contract_code: string
direction: 'buy' | 'sell'
offset: string
volume: number
price: number
created_at: number
}[]
}
type HuobiContractInfo = {
op: 'notify'
topic: string
ts: number
data: {
symbol: string
contract_code: string
contract_type: 'this_week' | 'next_week' | 'quarter' | 'next_quarter'
}[]
}
type HuobiOptionsOpenInterestMessage = {
ch: 'market.BTC-USDT-210521-C-42000.open_interest'
generated: true
data: [
{
volume: 684.0
amount: 0.684
symbol: 'BTC'
contract_type: 'this_week'
contract_code: 'BTC-USDT-210521-C-42000'
trade_partition: 'USDT'
trade_amount: 0.792
trade_volume: 792
trade_turnover: 3237.37806
}
]
ts: 1621296002336
}
type HuobiOptionsIndexMessage = {
ch: 'market.BTC-USDT.option_index'
generated: true
data: { symbol: 'BTC-USDT'; index_price: 43501.21; index_ts: 1621295997270 }
ts: 1621296002825
}
type HuobiOptionsMarketIndexMessage = {
ch: 'market.BTC-USDT-210521-P-42000.option_market_index'
generated: true
data: {
contract_code: 'BTC-USDT-210521-P-42000'
symbol: 'BTC'
iv_last_price: 1.62902357
iv_ask_one: 1.64869787
iv_bid_one: 1.13185884
iv_mark_price: 1.39190675
delta: -0.3704996546766173
gamma: 0.00006528
theta: -327.85540508
vega: 15.70293917
ask_one: 2000
bid_one: 1189.49
last_price: 1968.83
mark_price: 1594.739777491571343067
trade_partition: 'USDT'
contract_type: 'this_week'
option_right_type: 'P'
}
ts: 1621296002820
}
type HuobiBBOMessage =
| {
ch: 'market.BTC-USDT.bbo'
ts: 1630454400495
tick: {
mrid: 64797873746
id: 1630454400
bid: [47176.5, 1] | undefined
ask: [47176.6, 9249] | undefined
ts: 1630454400495
version: 64797873746
ch: 'market.BTC-USDT.bbo'
}
}
| {
ch: 'market.btcusdt.bbo'
ts: 1575158404058
tick: {
seqId: 103273695595
ask: 7543.59
askSize: 2.323241
bid: 7541.16
bidSize: 0.002329
quoteTime: number
symbol: 'btcusdt'
}
}
================================================
FILE: src/mappers/hyperliquid.ts
================================================
import { BookChange, BookTicker, DerivativeTicker, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
const KILO_SYMBOLS = ['kPEPE', 'kSHIB', 'kBONK', 'kFLOKI', 'kLUNC', 'kDOGS', 'kNEIRO']
function getApiSymbolId(symbol: string) {
const potentialKSymbol = symbol.charAt(0).toLowerCase() + symbol.slice(1)
if (KILO_SYMBOLS.includes(potentialKSymbol)) {
return potentialKSymbol
}
if (symbol.includes(':')) {
const [prefix, suffix] = symbol.split(':')
return prefix.toLowerCase() + ':' + suffix
}
return symbol
}
function getSymbols(symbols?: string[]) {
if (symbols !== undefined) {
return symbols.map(getApiSymbolId)
}
return
}
export class HyperliquidTradesMapper implements Mapper<'hyperliquid', Trade> {
private readonly _seenSymbols = new Set()
canHandle(message: HyperliquidTradeMessage) {
return message.channel === 'trades'
}
getFilters(symbols?: string[]) {
symbols = getSymbols(symbols)
return [
{
channel: 'trades',
symbols
}
]
}
*map(message: HyperliquidTradeMessage, localTimestamp: Date): IterableIterator {
for (const hyperliquidTrade of message.data) {
if (this._seenSymbols.has(hyperliquidTrade.coin) === false) {
this._seenSymbols.add(hyperliquidTrade.coin)
break
}
yield {
type: 'trade',
symbol: hyperliquidTrade.coin.toUpperCase(),
exchange: 'hyperliquid',
id: hyperliquidTrade.tid.toString(),
price: Number(hyperliquidTrade.px),
amount: Number(hyperliquidTrade.sz),
side: hyperliquidTrade.side === 'B' ? 'buy' : 'sell',
timestamp: new Date(hyperliquidTrade.time),
localTimestamp: localTimestamp
}
}
}
}
function mapHyperliquidLevel(level: HyperliquidWsLevel) {
return {
price: Number(level.px),
amount: Number(level.sz)
}
}
export class HyperliquidBookChangeMapper implements Mapper<'hyperliquid', BookChange> {
canHandle(message: HyperliquidWsBookMessage) {
return message.channel === 'l2Book'
}
getFilters(symbols?: string[]) {
symbols = getSymbols(symbols)
return [
{
channel: 'l2Book',
symbols
}
]
}
*map(message: HyperliquidWsBookMessage, localTimestamp: Date): IterableIterator {
yield {
type: 'book_change',
symbol: message.data.coin.toUpperCase(),
exchange: 'hyperliquid',
isSnapshot: true,
bids: (message.data.levels[0] ? message.data.levels[0] : []).map(mapHyperliquidLevel),
asks: (message.data.levels[1] ? message.data.levels[1] : []).map(mapHyperliquidLevel),
timestamp: new Date(message.data.time),
localTimestamp
}
}
}
export class HyperliquidBookTickerMapper implements Mapper<'hyperliquid', BookTicker> {
canHandle(message: HyperliquidBboMessage) {
return message.channel === 'bbo'
}
getFilters(symbols?: string[]) {
symbols = getSymbols(symbols)
return [
{
channel: 'bbo',
symbols
}
]
}
*map(message: HyperliquidBboMessage, localTimestamp: Date): IterableIterator {
const bbo = message.data.bbo
const bestBid = bbo[0]
const bestAsk = bbo[1]
const ticker: BookTicker = {
type: 'book_ticker',
symbol: message.data.coin.toUpperCase(),
exchange: 'hyperliquid',
bidPrice: bestBid ? Number(bestBid.px) : undefined,
bidAmount: bestBid ? Number(bestBid.sz) : undefined,
askPrice: bestAsk ? Number(bestAsk.px) : undefined,
askAmount: bestAsk ? Number(bestAsk.sz) : undefined,
timestamp: new Date(message.data.time),
localTimestamp: localTimestamp
}
yield ticker
}
}
export class HyperliquidDerivativeTickerMapper implements Mapper<'hyperliquid', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
canHandle(message: HyperliquidContextMessage) {
return message.channel === 'activeAssetCtx'
}
getFilters(symbols?: string[]) {
symbols = getSymbols(symbols)
return [
{
channel: 'activeAssetCtx',
symbols
}
]
}
*map(message: HyperliquidContextMessage, localTimestamp: Date): IterableIterator {
const symbol = message.data.coin.toUpperCase()
const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(symbol, 'hyperliquid')
if (message.data.ctx.funding !== undefined) {
pendingTickerInfo.updateFundingRate(Number(message.data.ctx.funding))
}
if (message.data.ctx.markPx !== undefined) {
pendingTickerInfo.updateMarkPrice(Number(message.data.ctx.markPx))
}
if (message.data.ctx.openInterest !== undefined) {
pendingTickerInfo.updateOpenInterest(Number(message.data.ctx.openInterest))
}
if (message.data.ctx.oraclePx !== undefined) {
pendingTickerInfo.updateIndexPrice(Number(message.data.ctx.oraclePx))
}
if (pendingTickerInfo.hasChanged()) {
pendingTickerInfo.updateTimestamp(localTimestamp)
yield pendingTickerInfo.getSnapshot(localTimestamp)
}
}
}
type HyperliquidTradeMessage = {
channel: 'trades'
data: [
{
coin: string
side: string
px: string
sz: string
hash: string
time: number
tid: number // ID unique across all assets
}
]
}
type HyperliquidWsBookMessage = {
channel: 'l2Book'
data: {
coin: 'ATOM'
time: 1730160007687
levels: [HyperliquidWsLevel[], HyperliquidWsLevel[]]
}
}
type HyperliquidWsLevel = {
px: string // price
sz: string // size
n: number // number of orders
}
type HyperliquidBboMessage = {
channel: 'bbo'
data: {
coin: string
time: number
bbo: [HyperliquidWsLevel, HyperliquidWsLevel]
}
}
type HyperliquidContextMessage = {
channel: 'activeAssetCtx'
data: {
coin: 'RENDER'
ctx: {
funding: '0.0000125'
openInterest: '231067.2'
prevDayPx: '4.8744'
dayNtlVlm: '387891.57092'
premium: '0.0'
oraclePx: '4.9185'
markPx: '4.919'
midPx: '4.9183'
impactPxs: ['4.9176', '4.9191']
}
}
}
================================================
FILE: src/mappers/index.ts
================================================
import { ONE_SEC_IN_MS } from '../handy.ts'
import { BookChange, DerivativeTicker, Liquidation, OptionSummary, BookTicker, Trade } from '../types.ts'
import { AscendexBookChangeMapper, AscendexDerivativeTickerMapper, AscendexBookTickerMapper, AscendexTradesMapper } from './ascendex.ts'
import {
BinanceBookChangeMapper,
BinanceFuturesBookChangeMapper,
BinanceFuturesDerivativeTickerMapper,
BinanceLiquidationsMapper,
BinanceBookTickerMapper,
BinanceTradesMapper
} from './binance.ts'
import { binanceDexBookChangeMapper, binanceDexBookTickerMapper, binanceDexTradesMapper } from './binancedex.ts'
import {
BinanceEuropeanOptionsBookChangeMapper,
BinanceEuropeanOptionsBookChangeMapperV2,
BinanceEuropeanOptionsBookTickerMapper,
BinanceEuropeanOptionsTradesMapper,
BinanceEuropeanOptionsTradesMapperV2,
BinanceEuropeanOptionSummaryMapper,
BinanceEuropeanOptionSummaryMapperV2
} from './binanceeuropeanoptions.ts'
import {
BitfinexBookChangeMapper,
BitfinexDerivativeTickerMapper,
BitfinexLiquidationsMapper,
BitfinexBookTickerMapper,
BitfinexTradesMapper
} from './bitfinex.ts'
import { BitflyerBookChangeMapper, bitflyerBookTickerMapper, bitflyerTradesMapper } from './bitflyer.ts'
import { BitgetBookChangeMapper, BitgetBookTickerMapper, BitgetDerivativeTickerMapper, BitgetTradesMapper } from './bitget.ts'
import {
BitmexBookChangeMapper,
BitmexDerivativeTickerMapper,
bitmexLiquidationsMapper,
bitmexBookTickerMapper,
bitmexTradesMapper
} from './bitmex.ts'
import { BitnomialBookChangMapper, bitnomialTradesMapper } from './bitnomial.ts'
import { BitstampBookChangeMapper, bitstampTradesMapper } from './bitstamp.ts'
import { BlockchainComBookChangeMapper, BlockchainComTradesMapper } from './blockchaincom.ts'
import {
BybitBookChangeMapper,
BybitDerivativeTickerMapper,
BybitLiquidationsMapper,
BybitTradesMapper,
BybitV5AllLiquidationsMapper,
BybitV5BookChangeMapper,
BybitV5BookTickerMapper,
BybitV5DerivativeTickerMapper,
BybitV5LiquidationsMapper,
BybitV5OptionSummaryMapper,
BybitV5TradesMapper
} from './bybit.ts'
import { BybitSpotBookChangeMapper, BybitSpotBookTickerMapper, BybitSpotTradesMapper } from './bybitspot.ts'
import { CoinbaseBookChangMapper, coinbaseBookTickerMapper, coinbaseTradesMapper } from './coinbase.ts'
import {
CoinbaseInternationalBookChangMapper,
coinbaseInternationalBookTickerMapper,
CoinbaseInternationalDerivativeTickerMapper,
coinbaseInternationalTradesMapper
} from './coinbaseinternational.ts'
import { coinflexBookChangeMapper, CoinflexDerivativeTickerMapper, coinflexTradesMapper } from './coinflex.ts'
import {
CryptoComBookChangeMapper,
CryptoComBookTickerMapper,
CryptoComDerivativeTickerMapper,
CryptoComTradesMapper
} from './cryptocom.ts'
import {
cryptofacilitiesBookChangeMapper,
CryptofacilitiesDerivativeTickerMapper,
cryptofacilitiesLiquidationsMapper,
cryptofacilitiesBookTickerMapper,
cryptofacilitiesTradesMapper
} from './cryptofacilities.ts'
import { DeltaBookChangeMapper, DeltaBookTickerMapper, DeltaDerivativeTickerMapper, DeltaTradesMapper } from './delta.ts'
import {
deribitBookChangeMapper,
DeribitDerivativeTickerMapper,
deribitLiquidationsMapper,
DeribitOptionSummaryMapper,
deribitBookTickerMapper,
deribitTradesMapper
} from './deribit.ts'
import { DydxBookChangeMapper, DydxDerivativeTickerMapper, DydxTradesMapper } from './dydx.ts'
import { DydxV4BookChangeMapper, DydxV4DerivativeTickerMapper, DydxV4LiquidationsMapper, DydxV4TradesMapper } from './dydxv4.ts'
import { FTXBookChangeMapper, FTXDerivativeTickerMapper, FTXLiquidationsMapper, FTXBookTickerMapper, FTXTradesMapper } from './ftx.ts'
import {
GateIOBookChangeMapper,
GateIOTradesMapper,
GateIOV4BookChangeMapper,
GateIOV4BookTickerMapper,
GateIOV4OrderBookV2ChangeMapper,
GateIOV4TradesMapper
} from './gateio.ts'
import {
GateIOFuturesBookChangeMapper,
GateIOFuturesBookTickerMapper,
GateIOFuturesDerivativeTickerMapper,
GateIOFuturesTradesMapper
} from './gateiofutures.ts'
import { geminiBookChangeMapper, geminiTradesMapper } from './gemini.ts'
import { hitBtcBookChangeMapper, hitBtcTradesMapper } from './hitbtc.ts'
import {
HuobiBookChangeMapper,
HuobiDerivativeTickerMapper,
HuobiLiquidationsMapper,
HuobiMBPBookChangeMapper,
HuobiOptionsSummaryMapper,
HuobiBookTickerMapper,
HuobiTradesMapper
} from './huobi.ts'
import {
HyperliquidBookChangeMapper,
HyperliquidBookTickerMapper,
HyperliquidDerivativeTickerMapper,
HyperliquidTradesMapper
} from './hyperliquid.ts'
import { krakenBookChangeMapper, krakenBookTickerMapper, krakenTradesMapper } from './kraken.ts'
import { KucoinBookChangeMapper, KucoinBookTickerMapper, KucoinTradesMapper } from './kucoin.ts'
import {
KucoinFuturesBookChangeMapper,
KucoinFuturesBookTickerMapper,
KucoinFuturesDerivativeTickerMapper,
KucoinFuturesTradesMapper
} from './kucoinfutures.ts'
import { Mapper } from './mapper.ts'
import {
OkexBookChangeMapper,
OkexBookTickerMapper,
OkexDerivativeTickerMapper,
OkexLiquidationsMapper,
OkexOptionSummaryMapper,
OkexTradesMapper,
OkexV5BookChangeMapper,
OkexV5BookTickerMapper,
OkexV5DerivativeTickerMapper,
OkexV5LiquidationsMapper,
OkexV5OptionSummaryMapper,
OkexV5TradesMapper
} from './okex.ts'
import { OkexSpreadsBookChangeMapper, OkexSpreadsBookTickerMapper, OkexSpreadsTradesMapper } from './okexspreads.ts'
import { phemexBookChangeMapper, PhemexDerivativeTickerMapper, phemexTradesMapper } from './phemex.ts'
import { PoloniexBookChangeMapper, PoloniexTradesMapper, PoloniexV2BookChangeMapper, PoloniexV2TradesMapper } from './poloniex.ts'
import { SerumBookChangeMapper, SerumBookTickerMapper, SerumTradesMapper } from './serum.ts'
import { UpbitBookChangeMapper, UpbitTradesMapper } from './upbit.ts'
import { WooxBookChangeMapper, WooxBookTickerMapper, WooxDerivativeTickerMapper, wooxTradesMapper } from './woox.ts'
export * from './mapper.ts'
const THREE_MINUTES_IN_MS = 3 * 60 * ONE_SEC_IN_MS
const isRealTime = (date: Date) => {
if (process.env.__NO_REAL_TIME__) {
return false
}
return date.valueOf() + THREE_MINUTES_IN_MS > new Date().valueOf()
}
const OKEX_V5_API_SWITCH_DATE = new Date('2021-12-23T00:00:00.000Z')
const OKEX_V5_TBT_BOOK_TICKER_RELEASE_DATE = new Date('2022-05-06T00:00:00.000Z')
const shouldUseOkexV5Mappers = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= OKEX_V5_API_SWITCH_DATE.valueOf()
}
const canUseOkexTbtBookTicker = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= OKEX_V5_TBT_BOOK_TICKER_RELEASE_DATE.valueOf()
}
const POLONIEX_V2_API_SWITCH_DATE = new Date('2022-08-02T00:00:00.000Z')
const shouldUsePoloniexV2Mappers = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= POLONIEX_V2_API_SWITCH_DATE.valueOf()
}
// see https://status.tardis.dev/incidents/ryjyv8tgdgkj
const shouldUseOKXPublicBooksChannel = (localTimestamp: Date) => {
return (
localTimestamp.valueOf() >= new Date('2023-02-25T00:00:00.000Z').valueOf() &&
localTimestamp.valueOf() < new Date('2023-03-09T00:00:00.000Z').valueOf()
)
}
const shouldIgnoreBookSnapshotOverlap = (date: Date) => {
if (process.env.IGNORE_BOOK_SNAPSHOT_OVERLAP_ERROR) {
return true
}
return isRealTime(date) === false
}
const BYBIT_V5_API_SWITCH_DATE = new Date('2023-04-05T00:00:00.000Z')
const BYBIT_V5_API_ALL_LIQUIDATION_SUPPORT_DATE = new Date('2025-02-26T00:00:00.000Z')
const shouldUseBybitV5Mappers = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= BYBIT_V5_API_SWITCH_DATE.valueOf()
}
const shouldUseBybitAllLiquidationFeed = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= BYBIT_V5_API_ALL_LIQUIDATION_SUPPORT_DATE.valueOf()
}
const OKCOIN_V5_API_SWITCH_DATE = new Date('2023-04-27T00:00:00.000Z')
const shouldUseOkcoinV5Mappers = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= OKCOIN_V5_API_SWITCH_DATE.valueOf()
}
const GATE_IO_V4_API_SWITCH_DATE = new Date('2023-04-29T00:00:00.000Z')
const GATE_IO_V4_ORDER_BOOK_V2_SWITCH_DATE = new Date('2025-08-01T00:00:00.000Z')
const shouldUseGateIOV4Mappers = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= GATE_IO_V4_API_SWITCH_DATE.valueOf()
}
const shouldUseGateIOV4OrderBookV2Mappers = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= GATE_IO_V4_ORDER_BOOK_V2_SWITCH_DATE.valueOf()
}
const shouldUseCFRelativeFunding = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= new Date('2022-09-29T00:00:00.000Z').valueOf()
}
const shouldUseOKXTradesAllChannel = (localTimestamp: Date) => {
if (process.env.OKX_USE_TRADES_CHANNEL) {
return false
}
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= new Date('2023-10-19T00:00:00.000Z').valueOf()
}
const BINANCE_EUROPEAN_OPTIONS_V2_API_SWITCH_DATE = new Date('2025-12-17T00:00:00.000Z')
const shouldUseBinanceEuropeanOptionsV2Mappers = (localTimestamp: Date) => {
return isRealTime(localTimestamp) || localTimestamp.valueOf() >= BINANCE_EUROPEAN_OPTIONS_V2_API_SWITCH_DATE.valueOf()
}
const tradesMappers = {
bitmex: () => bitmexTradesMapper,
binance: () => new BinanceTradesMapper('binance'),
'binance-us': () => new BinanceTradesMapper('binance-us'),
'binance-jersey': () => new BinanceTradesMapper('binance-jersey'),
'binance-futures': () => new BinanceTradesMapper('binance-futures'),
'binance-delivery': () => new BinanceTradesMapper('binance-delivery'),
'binance-dex': () => binanceDexTradesMapper,
bitfinex: () => new BitfinexTradesMapper('bitfinex'),
'bitfinex-derivatives': () => new BitfinexTradesMapper('bitfinex-derivatives'),
bitflyer: () => bitflyerTradesMapper,
bitstamp: () => bitstampTradesMapper,
coinbase: () => coinbaseTradesMapper,
cryptofacilities: () => cryptofacilitiesTradesMapper,
deribit: () => deribitTradesMapper,
ftx: () => new FTXTradesMapper('ftx'),
'ftx-us': () => new FTXTradesMapper('ftx-us'),
gemini: () => geminiTradesMapper,
kraken: () => krakenTradesMapper,
okex: (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5TradesMapper('okex', shouldUseOKXTradesAllChannel(localTimestamp))
: new OkexTradesMapper('okex', 'spot'),
'okex-futures': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5TradesMapper('okex-futures', shouldUseOKXTradesAllChannel(localTimestamp))
: new OkexTradesMapper('okex-futures', 'futures'),
'okex-swap': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5TradesMapper('okex-swap', shouldUseOKXTradesAllChannel(localTimestamp))
: new OkexTradesMapper('okex-swap', 'swap'),
'okex-options': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5TradesMapper('okex-options', shouldUseOKXTradesAllChannel(localTimestamp))
: new OkexTradesMapper('okex-options', 'option'),
huobi: () => new HuobiTradesMapper('huobi'),
'huobi-dm': () => new HuobiTradesMapper('huobi-dm'),
'huobi-dm-swap': () => new HuobiTradesMapper('huobi-dm-swap'),
'huobi-dm-linear-swap': () => new HuobiTradesMapper('huobi-dm-linear-swap'),
'huobi-dm-options': () => new HuobiTradesMapper('huobi-dm-options'),
bybit: (localTimestamp: Date) =>
shouldUseBybitV5Mappers(localTimestamp) ? new BybitV5TradesMapper('bybit') : new BybitTradesMapper('bybit'),
okcoin: (localTimestamp: Date) =>
shouldUseOkcoinV5Mappers(localTimestamp) ? new OkexV5TradesMapper('okcoin', false) : new OkexTradesMapper('okcoin', 'spot'),
hitbtc: () => hitBtcTradesMapper,
phemex: () => phemexTradesMapper,
delta: (localTimestamp: Date) => new DeltaTradesMapper(localTimestamp.valueOf() >= new Date('2020-10-14').valueOf()),
'gate-io': (localTimestamp: Date) =>
shouldUseGateIOV4Mappers(localTimestamp) ? new GateIOV4TradesMapper('gate-io') : new GateIOTradesMapper('gate-io'),
'gate-io-futures': () => new GateIOFuturesTradesMapper('gate-io-futures'),
poloniex: (localTimestamp: Date) =>
shouldUsePoloniexV2Mappers(localTimestamp) ? new PoloniexV2TradesMapper() : new PoloniexTradesMapper(),
coinflex: () => coinflexTradesMapper,
upbit: () => new UpbitTradesMapper(),
ascendex: () => new AscendexTradesMapper(),
dydx: () => new DydxTradesMapper(),
'dydx-v4': () => new DydxV4TradesMapper(),
serum: () => new SerumTradesMapper('serum'),
'star-atlas': () => new SerumTradesMapper('star-atlas'),
mango: () => new SerumTradesMapper('mango'),
'bybit-spot': (localTimestamp: Date) =>
shouldUseBybitV5Mappers(localTimestamp) ? new BybitV5TradesMapper('bybit-spot') : new BybitSpotTradesMapper('bybit-spot'),
'crypto-com': () => new CryptoComTradesMapper('crypto-com'),
kucoin: () => new KucoinTradesMapper('kucoin'),
'kucoin-futures': () => new KucoinFuturesTradesMapper(),
bitnomial: () => bitnomialTradesMapper,
'woo-x': () => wooxTradesMapper,
'blockchain-com': () => new BlockchainComTradesMapper(),
'bybit-options': () => new BybitV5TradesMapper('bybit-options'),
'binance-european-options': (localTimestamp: Date) =>
shouldUseBinanceEuropeanOptionsV2Mappers(localTimestamp)
? new BinanceEuropeanOptionsTradesMapperV2()
: new BinanceEuropeanOptionsTradesMapper(),
'okex-spreads': () => new OkexSpreadsTradesMapper(),
bitget: () => new BitgetTradesMapper('bitget'),
'bitget-futures': () => new BitgetTradesMapper('bitget-futures'),
'coinbase-international': () => coinbaseInternationalTradesMapper,
hyperliquid: () => new HyperliquidTradesMapper()
}
const bookChangeMappers = {
bitmex: () => new BitmexBookChangeMapper(),
binance: (localTimestamp: Date) => new BinanceBookChangeMapper('binance', shouldIgnoreBookSnapshotOverlap(localTimestamp)),
'binance-us': (localTimestamp: Date) => new BinanceBookChangeMapper('binance-us', shouldIgnoreBookSnapshotOverlap(localTimestamp)),
'binance-jersey': (localTimestamp: Date) =>
new BinanceBookChangeMapper('binance-jersey', shouldIgnoreBookSnapshotOverlap(localTimestamp)),
'binance-futures': (localTimestamp: Date) =>
new BinanceFuturesBookChangeMapper('binance-futures', shouldIgnoreBookSnapshotOverlap(localTimestamp)),
'binance-delivery': (localTimestamp: Date) =>
new BinanceFuturesBookChangeMapper('binance-delivery', shouldIgnoreBookSnapshotOverlap(localTimestamp)),
'binance-dex': () => binanceDexBookChangeMapper,
bitfinex: () => new BitfinexBookChangeMapper('bitfinex'),
'bitfinex-derivatives': () => new BitfinexBookChangeMapper('bitfinex-derivatives'),
bitflyer: () => new BitflyerBookChangeMapper(),
bitstamp: () => new BitstampBookChangeMapper(),
coinbase: () => new CoinbaseBookChangMapper(),
cryptofacilities: () => cryptofacilitiesBookChangeMapper,
deribit: () => deribitBookChangeMapper,
ftx: () => new FTXBookChangeMapper('ftx'),
'ftx-us': () => new FTXBookChangeMapper('ftx-us'),
gemini: () => geminiBookChangeMapper,
kraken: () => krakenBookChangeMapper,
okex: (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5BookChangeMapper('okex', isRealTime(localTimestamp) || shouldUseOKXPublicBooksChannel(localTimestamp))
: new OkexBookChangeMapper('okex', 'spot', localTimestamp.valueOf() >= new Date('2020-04-10').valueOf()),
'okex-futures': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5BookChangeMapper('okex-futures', isRealTime(localTimestamp) || shouldUseOKXPublicBooksChannel(localTimestamp))
: new OkexBookChangeMapper('okex-futures', 'futures', localTimestamp.valueOf() >= new Date('2019-12-05').valueOf()),
'okex-swap': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5BookChangeMapper('okex-swap', isRealTime(localTimestamp) || shouldUseOKXPublicBooksChannel(localTimestamp))
: new OkexBookChangeMapper('okex-swap', 'swap', localTimestamp.valueOf() >= new Date('2020-02-08').valueOf()),
'okex-options': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5BookChangeMapper('okex-options', isRealTime(localTimestamp) || shouldUseOKXPublicBooksChannel(localTimestamp))
: new OkexBookChangeMapper('okex-options', 'option', localTimestamp.valueOf() >= new Date('2020-02-08').valueOf()),
huobi: (localTimestamp: Date) =>
localTimestamp.valueOf() >= new Date('2020-07-03').valueOf()
? new HuobiMBPBookChangeMapper('huobi')
: new HuobiBookChangeMapper('huobi'),
'huobi-dm': () => new HuobiBookChangeMapper('huobi-dm'),
'huobi-dm-swap': () => new HuobiBookChangeMapper('huobi-dm-swap'),
'huobi-dm-linear-swap': () => new HuobiBookChangeMapper('huobi-dm-linear-swap'),
'huobi-dm-options': () => new HuobiBookChangeMapper('huobi-dm-options'),
'bybit-spot': (localTimestamp: Date) =>
shouldUseBybitV5Mappers(localTimestamp) ? new BybitV5BookChangeMapper('bybit-spot', 50) : new BybitSpotBookChangeMapper('bybit-spot'),
bybit: (localTimestamp: Date) =>
shouldUseBybitV5Mappers(localTimestamp) ? new BybitV5BookChangeMapper('bybit', 50) : new BybitBookChangeMapper('bybit', false),
okcoin: (localTimestamp: Date) =>
shouldUseOkcoinV5Mappers(localTimestamp)
? new OkexV5BookChangeMapper('okcoin', true)
: new OkexBookChangeMapper('okcoin', 'spot', localTimestamp.valueOf() >= new Date('2020-02-13').valueOf()),
hitbtc: () => hitBtcBookChangeMapper,
phemex: () => phemexBookChangeMapper,
delta: (localTimestamp: Date) => new DeltaBookChangeMapper(localTimestamp.valueOf() >= new Date('2023-04-01').valueOf()),
'gate-io': (localTimestamp: Date) =>
shouldUseGateIOV4OrderBookV2Mappers(localTimestamp)
? new GateIOV4OrderBookV2ChangeMapper('gate-io')
: shouldUseGateIOV4Mappers(localTimestamp)
? new GateIOV4BookChangeMapper('gate-io', isRealTime(localTimestamp) == false)
: new GateIOBookChangeMapper('gate-io'),
'gate-io-futures': () => new GateIOFuturesBookChangeMapper('gate-io-futures'),
poloniex: (localTimestamp: Date) =>
shouldUsePoloniexV2Mappers(localTimestamp) ? new PoloniexV2BookChangeMapper() : new PoloniexBookChangeMapper(),
coinflex: () => coinflexBookChangeMapper,
upbit: () => new UpbitBookChangeMapper(),
ascendex: () => new AscendexBookChangeMapper(),
dydx: () => new DydxBookChangeMapper(),
'dydx-v4': () => new DydxV4BookChangeMapper(),
serum: () => new SerumBookChangeMapper('serum'),
'star-atlas': () => new SerumBookChangeMapper('star-atlas'),
mango: () => new SerumBookChangeMapper('mango'),
'crypto-com': () => new CryptoComBookChangeMapper('crypto-com'),
kucoin: (localTimestamp: Date) => new KucoinBookChangeMapper('kucoin', isRealTime(localTimestamp) === false),
'kucoin-futures': (localTimestamp: Date) => new KucoinFuturesBookChangeMapper(isRealTime(localTimestamp) === false),
bitnomial: () => new BitnomialBookChangMapper(),
'woo-x': () => new WooxBookChangeMapper(),
'blockchain-com': () => new BlockchainComBookChangeMapper(),
'bybit-options': () => new BybitV5BookChangeMapper('bybit-options', 25),
'binance-european-options': (localTimestamp: Date) =>
shouldUseBinanceEuropeanOptionsV2Mappers(localTimestamp)
? new BinanceEuropeanOptionsBookChangeMapperV2()
: new BinanceEuropeanOptionsBookChangeMapper(),
'okex-spreads': () => new OkexSpreadsBookChangeMapper(),
bitget: () => new BitgetBookChangeMapper('bitget'),
'bitget-futures': () => new BitgetBookChangeMapper('bitget-futures'),
'coinbase-international': () => new CoinbaseInternationalBookChangMapper(),
hyperliquid: () => new HyperliquidBookChangeMapper()
}
const derivativeTickersMappers = {
bitmex: () => new BitmexDerivativeTickerMapper(),
'binance-futures': () => new BinanceFuturesDerivativeTickerMapper('binance-futures'),
'binance-delivery': () => new BinanceFuturesDerivativeTickerMapper('binance-delivery'),
'bitfinex-derivatives': () => new BitfinexDerivativeTickerMapper(),
cryptofacilities: (localTimestamp: Date) => new CryptofacilitiesDerivativeTickerMapper(shouldUseCFRelativeFunding(localTimestamp)),
deribit: () => new DeribitDerivativeTickerMapper(),
'okex-futures': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5DerivativeTickerMapper('okex-futures')
: new OkexDerivativeTickerMapper('okex-futures'),
'okex-swap': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp) ? new OkexV5DerivativeTickerMapper('okex-swap') : new OkexDerivativeTickerMapper('okex-swap'),
bybit: (localTimestamp: Date) =>
shouldUseBybitV5Mappers(localTimestamp) ? new BybitV5DerivativeTickerMapper() : new BybitDerivativeTickerMapper(),
phemex: () => new PhemexDerivativeTickerMapper(),
ftx: () => new FTXDerivativeTickerMapper('ftx'),
delta: (localTimestamp: Date) => new DeltaDerivativeTickerMapper(localTimestamp.valueOf() >= new Date('2020-10-14').valueOf()),
'huobi-dm': () => new HuobiDerivativeTickerMapper('huobi-dm'),
'huobi-dm-swap': () => new HuobiDerivativeTickerMapper('huobi-dm-swap'),
'huobi-dm-linear-swap': () => new HuobiDerivativeTickerMapper('huobi-dm-linear-swap'),
'gate-io-futures': () => new GateIOFuturesDerivativeTickerMapper(),
coinflex: () => new CoinflexDerivativeTickerMapper(),
ascendex: () => new AscendexDerivativeTickerMapper(),
dydx: () => new DydxDerivativeTickerMapper(),
'dydx-v4': () => new DydxV4DerivativeTickerMapper(),
'crypto-com': () => new CryptoComDerivativeTickerMapper('crypto-com'),
'woo-x': () => new WooxDerivativeTickerMapper(),
'kucoin-futures': () => new KucoinFuturesDerivativeTickerMapper(),
'bitget-futures': () => new BitgetDerivativeTickerMapper(),
'coinbase-international': () => new CoinbaseInternationalDerivativeTickerMapper(),
hyperliquid: () => new HyperliquidDerivativeTickerMapper()
}
const optionsSummaryMappers = {
deribit: () => new DeribitOptionSummaryMapper(),
'okex-options': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp) ? new OkexV5OptionSummaryMapper() : new OkexOptionSummaryMapper(),
'huobi-dm-options': () => new HuobiOptionsSummaryMapper(),
'bybit-options': () => new BybitV5OptionSummaryMapper(),
'binance-european-options': (localTimestamp: Date) =>
shouldUseBinanceEuropeanOptionsV2Mappers(localTimestamp)
? new BinanceEuropeanOptionSummaryMapperV2()
: new BinanceEuropeanOptionSummaryMapper()
}
const liquidationsMappers = {
ftx: () => new FTXLiquidationsMapper(),
bitmex: () => bitmexLiquidationsMapper,
deribit: () => deribitLiquidationsMapper,
'binance-futures': () => new BinanceLiquidationsMapper('binance-futures'),
'binance-delivery': () => new BinanceLiquidationsMapper('binance-delivery'),
'bitfinex-derivatives': () => new BitfinexLiquidationsMapper('bitfinex-derivatives'),
cryptofacilities: () => cryptofacilitiesLiquidationsMapper,
'huobi-dm': () => new HuobiLiquidationsMapper('huobi-dm'),
'dydx-v4': () => new DydxV4LiquidationsMapper(),
'huobi-dm-swap': () => new HuobiLiquidationsMapper('huobi-dm-swap'),
'huobi-dm-linear-swap': () => new HuobiLiquidationsMapper('huobi-dm-linear-swap'),
bybit: (localTimestamp: Date) =>
shouldUseBybitV5Mappers(localTimestamp)
? shouldUseBybitAllLiquidationFeed(localTimestamp)
? new BybitV5AllLiquidationsMapper('bybit')
: new BybitV5LiquidationsMapper('bybit')
: new BybitLiquidationsMapper('bybit'),
'okex-futures': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5LiquidationsMapper('okex-futures')
: new OkexLiquidationsMapper('okex-futures', 'futures'),
'okex-swap': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp) ? new OkexV5LiquidationsMapper('okex-swap') : new OkexLiquidationsMapper('okex-swap', 'swap')
}
const bookTickersMappers = {
binance: () => new BinanceBookTickerMapper('binance'),
'binance-futures': () => new BinanceBookTickerMapper('binance-futures'),
'binance-delivery': () => new BinanceBookTickerMapper('binance-delivery'),
'binance-us': () => new BinanceBookTickerMapper('binance-us'),
ascendex: () => new AscendexBookTickerMapper(),
'binance-dex': () => binanceDexBookTickerMapper,
bitfinex: () => new BitfinexBookTickerMapper('bitfinex'),
'bitfinex-derivatives': () => new BitfinexBookTickerMapper('bitfinex-derivatives'),
bitflyer: () => bitflyerBookTickerMapper,
bitmex: () => bitmexBookTickerMapper,
coinbase: () => coinbaseBookTickerMapper,
cryptofacilities: () => cryptofacilitiesBookTickerMapper,
deribit: () => deribitBookTickerMapper,
ftx: () => new FTXBookTickerMapper('ftx'),
'ftx-us': () => new FTXBookTickerMapper('ftx-us'),
huobi: () => new HuobiBookTickerMapper('huobi'),
'huobi-dm': () => new HuobiBookTickerMapper('huobi-dm'),
'huobi-dm-swap': () => new HuobiBookTickerMapper('huobi-dm-swap'),
'huobi-dm-linear-swap': () => new HuobiBookTickerMapper('huobi-dm-linear-swap'),
kraken: () => krakenBookTickerMapper,
okex: (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5BookTickerMapper('okex', canUseOkexTbtBookTicker(localTimestamp))
: new OkexBookTickerMapper('okex', 'spot'),
'okex-futures': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5BookTickerMapper('okex-futures', canUseOkexTbtBookTicker(localTimestamp))
: new OkexBookTickerMapper('okex-futures', 'futures'),
'okex-swap': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5BookTickerMapper('okex-swap', canUseOkexTbtBookTicker(localTimestamp))
: new OkexBookTickerMapper('okex-swap', 'swap'),
'okex-options': (localTimestamp: Date) =>
shouldUseOkexV5Mappers(localTimestamp)
? new OkexV5BookTickerMapper('okex-options', canUseOkexTbtBookTicker(localTimestamp))
: new OkexBookTickerMapper('okex-options', 'option'),
okcoin: (localTimestamp: Date) =>
shouldUseOkcoinV5Mappers(localTimestamp) ? new OkexV5BookTickerMapper('okcoin', true) : new OkexBookTickerMapper('okcoin', 'spot'),
serum: () => new SerumBookTickerMapper('serum'),
'star-atlas': () => new SerumBookTickerMapper('star-atlas'),
mango: () => new SerumBookTickerMapper('mango'),
'gate-io-futures': () => new GateIOFuturesBookTickerMapper('gate-io-futures'),
'bybit-spot': (localTimestamp: Date) =>
shouldUseBybitV5Mappers(localTimestamp) ? new BybitV5BookTickerMapper('bybit-spot') : new BybitSpotBookTickerMapper('bybit-spot'),
'crypto-com': () => new CryptoComBookTickerMapper('crypto-com'),
kucoin: () => new KucoinBookTickerMapper('kucoin'),
'woo-x': () => new WooxBookTickerMapper(),
delta: () => new DeltaBookTickerMapper(),
bybit: () => new BybitV5BookTickerMapper('bybit'),
'gate-io': () => new GateIOV4BookTickerMapper('gate-io'),
'okex-spreads': () => new OkexSpreadsBookTickerMapper(),
'kucoin-futures': () => new KucoinFuturesBookTickerMapper(),
bitget: () => new BitgetBookTickerMapper('bitget'),
'bitget-futures': () => new BitgetBookTickerMapper('bitget-futures'),
'coinbase-international': () => coinbaseInternationalBookTickerMapper,
hyperliquid: () => new HyperliquidBookTickerMapper(),
'binance-european-options': () => new BinanceEuropeanOptionsBookTickerMapper()
}
export const normalizeTrades = (exchange: T, localTimestamp: Date): Mapper => {
const createTradesMapper = tradesMappers[exchange]
if (createTradesMapper === undefined) {
throw new Error(`normalizeTrades: ${exchange} not supported`)
}
return createTradesMapper(localTimestamp) as Mapper
}
export const normalizeBookChanges = (
exchange: T,
localTimestamp: Date
): Mapper => {
const createBookChangesMapper = bookChangeMappers[exchange]
if (createBookChangesMapper === undefined) {
throw new Error(`normalizeBookChanges: ${exchange} not supported`)
}
return createBookChangesMapper(localTimestamp) as Mapper
}
export const normalizeDerivativeTickers = (
exchange: T,
localTimestamp: Date
): Mapper => {
const createDerivativeTickerMapper = derivativeTickersMappers[exchange]
if (createDerivativeTickerMapper === undefined) {
throw new Error(`normalizeDerivativeTickers: ${exchange} not supported`)
}
return createDerivativeTickerMapper(localTimestamp) as any
}
export const normalizeOptionsSummary = (
exchange: T,
localTimestamp: Date
): Mapper => {
const createOptionSummaryMapper = optionsSummaryMappers[exchange]
if (createOptionSummaryMapper === undefined) {
throw new Error(`normalizeOptionsSummary: ${exchange} not supported`)
}
return createOptionSummaryMapper(localTimestamp) as any
}
export const normalizeLiquidations = (
exchange: T,
localTimestamp: Date
): Mapper => {
const createLiquidationsMapper = liquidationsMappers[exchange]
if (createLiquidationsMapper === undefined) {
throw new Error(`normalizeLiquidations: ${exchange} not supported`)
}
return createLiquidationsMapper(localTimestamp) as any
}
export const normalizeBookTickers = (
exchange: T,
localTimestamp: Date
): Mapper => {
const createTickerMapper = bookTickersMappers[exchange]
if (createTickerMapper === undefined) {
throw new Error(`normalizeBookTickers: ${exchange} not supported`)
}
return createTickerMapper(localTimestamp) as any
}
================================================
FILE: src/mappers/kraken.ts
================================================
import { asNumberIfValid, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, Trade } from '../types.ts'
import { Mapper } from './mapper.ts'
// https://www.kraken.com/features/websocket-api
export const krakenTradesMapper: Mapper<'kraken', Trade> = {
canHandle(message: KrakenTrades) {
if (!Array.isArray(message)) {
return false
}
const channel = message[message.length - 2] as string
return channel === 'trade'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'trade',
symbols
}
]
},
*map(message: KrakenTrades, localTimestamp: Date): IterableIterator {
const [_, trades, __, symbol] = message
for (const [price, amount, time, side] of trades) {
const timeExchange = Number(time)
const timestamp = new Date(timeExchange * 1000)
timestamp.μs = Math.floor(timeExchange * 1000000) % 1000
yield {
type: 'trade',
symbol,
exchange: 'kraken',
id: undefined,
price: Number(price),
amount: Number(amount),
side: side === 'b' ? 'buy' : 'sell',
timestamp,
localTimestamp
}
}
}
}
const mapBookLevel = (level: KrakenBookLevel) => {
const [price, amount] = level
return { price: Number(price), amount: Number(amount) }
}
const getLatestTimestamp = (bids: KrakenBookLevel[], asks: KrakenBookLevel[]): Date => {
const timestampsSorted = [...bids.map((b) => Number(b[2])), ...asks.map((b) => Number(b[2]))].sort()
const lastBookUpdateTime = timestampsSorted[timestampsSorted.length - 1]
const timestamp = new Date(lastBookUpdateTime * 1000)
timestamp.μs = Math.floor(lastBookUpdateTime * 1000000) % 1000
return timestamp
}
export const krakenBookChangeMapper: Mapper<'kraken', BookChange> = {
canHandle(message: KrakenBookSnapshot | KrakenBookUpdate) {
if (!Array.isArray(message)) {
return false
}
const channel = message[message.length - 2] as string
return channel.startsWith('book')
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'book',
symbols
}
]
},
*map(message: KrakenBookSnapshot | KrakenBookUpdate, localTimestamp: Date): IterableIterator {
if ('as' in message[1]) {
// we've got snapshot message
const [_, { as, bs }, __, symbol] = message
yield {
type: 'book_change',
symbol: symbol,
exchange: 'kraken',
isSnapshot: true,
bids: bs.map(mapBookLevel),
asks: as.map(mapBookLevel),
timestamp: getLatestTimestamp(as, bs),
localTimestamp: localTimestamp
}
} else {
// we've got update message
const symbol = message[message.length - 1] as string
const asks = 'a' in message[1] ? message[1].a : []
const bids = 'b' in message[1] ? message[1].b : typeof message[2] !== 'string' && 'b' in message[2] ? message[2].b : []
yield {
type: 'book_change',
symbol,
exchange: 'kraken',
isSnapshot: false,
bids: bids.map(mapBookLevel),
asks: asks.map(mapBookLevel),
timestamp: getLatestTimestamp(asks, bids),
localTimestamp: localTimestamp
}
}
}
}
export const krakenBookTickerMapper: Mapper<'kraken', BookTicker> = {
canHandle(message: KrakenSpread) {
if (!Array.isArray(message)) {
return false
}
const channel = message[message.length - 2] as string
return channel === 'spread'
},
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'spread',
symbols
}
]
},
*map(message: KrakenSpread, localTimestamp: Date): IterableIterator {
const [bid, ask, time, bidVolume, askVolume] = message[1]
const timeExchange = Number(time)
if (timeExchange === 0) {
return
}
const timestamp = new Date(timeExchange * 1000)
timestamp.μs = Math.floor(timeExchange * 1000000) % 1000
const ticker: BookTicker = {
type: 'book_ticker',
symbol: message[3],
exchange: 'kraken',
askAmount: asNumberIfValid(askVolume),
askPrice: asNumberIfValid(ask),
bidPrice: asNumberIfValid(bid),
bidAmount: asNumberIfValid(bidVolume),
timestamp,
localTimestamp: localTimestamp
}
yield ticker
}
}
type KrakenTrades = [number, [string, string, string, 's' | 'b', string, string][], string, string]
type KrakenBookLevel = [string, string, string]
type KrakenBookSnapshot = [
number,
{
as: KrakenBookLevel[]
bs: KrakenBookLevel[]
},
string,
string
]
type KrakenBookUpdate =
| [
number,
(
| {
a: KrakenBookLevel[]
}
| {
b: KrakenBookLevel[]
}
),
string,
string
]
| [
number,
{
a: KrakenBookLevel[]
},
{
b: KrakenBookLevel[]
},
string,
string
]
type KrakenSpread = [
325,
[bid: '43770.20000', ask: '43770.30000', timestamp: '1633053779.916349', bidVolume: '0.00917717', askVolume: '0.31670440'],
'spread',
'XBT/USD'
]
================================================
FILE: src/mappers/kucoin.ts
================================================
import { debug } from '../debug.ts'
import { CircularBuffer, upperCaseSymbols } from '../handy.ts'
import { BookChange, Exchange, BookTicker, Trade, BookPriceLevel } from '../types.ts'
import { Mapper } from './mapper.ts'
export class KucoinTradesMapper implements Mapper<'kucoin', Trade> {
constructor(private readonly _exchange: Exchange) {}
canHandle(message: KucoinTradeMessage) {
return message.type === 'message' && message.topic.startsWith('/market/match')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'market/match',
symbols
} as const
]
}
*map(message: KucoinTradeMessage, localTimestamp: Date): IterableIterator {
const kucoinTrade = message.data
const timestamp = new Date(Number(kucoinTrade.time.slice(0, 13)))
timestamp.μs = Number(kucoinTrade.time.slice(13, 16))
yield {
type: 'trade',
symbol: kucoinTrade.symbol,
exchange: this._exchange,
id: kucoinTrade.tradeId,
price: Number(kucoinTrade.price),
amount: Number(kucoinTrade.size),
side: kucoinTrade.side === 'sell' ? 'sell' : 'buy',
timestamp,
localTimestamp
}
}
}
export class KucoinBookChangeMapper implements Mapper<'kucoin', BookChange> {
protected readonly symbolToDepthInfoMapping: {
[key: string]: LocalDepthInfo
} = {}
constructor(protected readonly _exchange: Exchange, private readonly ignoreBookSnapshotOverlapError: boolean) {}
canHandle(message: KucoinLevel2SnapshotMessage | KucoinLevel2UpdateMessage) {
return message.type === 'message' && message.topic.startsWith('/market/level2')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'market/level2',
symbols
} as const,
{
channel: 'market/level2Snapshot',
symbols
} as const
]
}
*map(message: KucoinLevel2SnapshotMessage | KucoinLevel2UpdateMessage, localTimestamp: Date) {
const symbol = message.topic.split(':')[1]
if (this.symbolToDepthInfoMapping[symbol] === undefined) {
this.symbolToDepthInfoMapping[symbol] = {
bufferedUpdates: new CircularBuffer(2000)
}
}
const symbolDepthInfo = this.symbolToDepthInfoMapping[symbol]
const snapshotAlreadyProcessed = symbolDepthInfo.snapshotProcessed
// first check if received message is snapshot and process it as such if it is
if (message.subject === 'trade.l2Snapshot') {
// if we've already received 'manual' snapshot, ignore if there is another one
if (snapshotAlreadyProcessed) {
return
}
// produce snapshot book_change
const kucoinSnapshotData = message.data
if (!kucoinSnapshotData.asks) {
kucoinSnapshotData.asks = []
}
if (!kucoinSnapshotData.bids) {
kucoinSnapshotData.bids = []
}
// mark given symbol depth info that has snapshot processed
symbolDepthInfo.lastUpdateId = Number(kucoinSnapshotData.sequence)
symbolDepthInfo.snapshotProcessed = true
// if there were any depth updates buffered, let's proccess those by adding to or updating the initial snapshot
for (const update of symbolDepthInfo.bufferedUpdates.items()) {
const bookChange = this.mapBookDepthUpdate(update, localTimestamp)
if (bookChange !== undefined) {
for (const bid of update.data.changes.bids) {
if (bid[0] == '0') {
continue
}
const matchingBid = kucoinSnapshotData.bids.find((b) => b[0] === bid[0])
if (matchingBid !== undefined) {
matchingBid[1] = bid[1]
} else {
kucoinSnapshotData.bids.push([bid[0], bid[1]])
}
}
for (const ask of update.data.changes.asks) {
if (ask[0] == '0') {
continue
}
const matchingAsk = kucoinSnapshotData.asks.find((a) => a[0] === ask[0])
if (matchingAsk !== undefined) {
matchingAsk[1] = ask[1]
} else {
kucoinSnapshotData.asks.push([ask[0], ask[1]])
}
}
}
}
// remove all buffered updates
symbolDepthInfo.bufferedUpdates.clear()
const bookChange: BookChange = {
type: 'book_change',
symbol,
exchange: this._exchange,
isSnapshot: true,
bids: kucoinSnapshotData.bids.map(this.mapBookLevel),
asks: kucoinSnapshotData.asks.map(this.mapBookLevel),
timestamp: localTimestamp,
localTimestamp
}
yield bookChange
} else if (snapshotAlreadyProcessed) {
// snapshot was already processed let's map the message as normal book_change
const bookChange = this.mapBookDepthUpdate(message, localTimestamp)
if (bookChange !== undefined) {
yield bookChange
}
} else {
symbolDepthInfo.bufferedUpdates.append(message)
}
}
protected mapBookDepthUpdate(l2UpdateMessage: KucoinLevel2UpdateMessage, localTimestamp: Date): BookChange | undefined {
// we can safely assume here that depthContext and lastUpdateId aren't null here as this is method only works
// when we've already processed the snapshot
const depthContext = this.symbolToDepthInfoMapping[l2UpdateMessage.data.symbol]!
const lastUpdateId = depthContext.lastUpdateId!
// Drop any event where sequenceEnd is <= lastUpdateId in the snapshot
if (l2UpdateMessage.data.sequenceEnd <= lastUpdateId) {
return
}
// The first processed event should have sequenceStart <= lastUpdateId+1 AND sequenceEnd >= lastUpdateId+1.
if (!depthContext.validatedFirstUpdate) {
// if there is new instrument added it can have empty book at first and that's normal
const bookSnapshotIsEmpty = lastUpdateId == -1 || lastUpdateId == 0
if (
(l2UpdateMessage.data.sequenceStart <= lastUpdateId + 1 && l2UpdateMessage.data.sequenceEnd >= lastUpdateId + 1) ||
bookSnapshotIsEmpty
) {
depthContext.validatedFirstUpdate = true
} else {
const message = `Book depth snapshot has no overlap with first update, update ${JSON.stringify(
l2UpdateMessage
)}, lastUpdateId: ${lastUpdateId}, exchange ${this._exchange}`
if (this.ignoreBookSnapshotOverlapError) {
depthContext.validatedFirstUpdate = true
debug(message)
} else {
throw new Error(message)
}
}
}
const bids = l2UpdateMessage.data.changes.bids.map(this.mapBookLevel).filter(this.nonZeroLevels)
const asks = l2UpdateMessage.data.changes.asks.map(this.mapBookLevel).filter(this.nonZeroLevels)
if (bids.length === 0 && asks.length === 0) {
return
}
const timestamp = l2UpdateMessage.data.time !== undefined ? new Date(l2UpdateMessage.data.time) : localTimestamp
return {
type: 'book_change',
symbol: l2UpdateMessage.data.symbol,
exchange: this._exchange,
isSnapshot: false,
bids,
asks,
timestamp: timestamp,
localTimestamp: localTimestamp
}
}
private mapBookLevel(level: [string, string, string?]) {
const price = Number(level[0])
const amount = Number(level[1])
return { price, amount }
}
private nonZeroLevels(level: BookPriceLevel) {
return level.price > 0
}
}
export class KucoinBookTickerMapper implements Mapper<'kucoin', BookTicker> {
constructor(protected readonly _exchange: Exchange) {}
canHandle(message: KucoinTickerMessage) {
return message.type === 'message' && message.topic.startsWith('/market/ticker')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'market/ticker',
symbols
} as const
]
}
*map(message: KucoinTickerMessage, localTimestamp: Date) {
const symbol = message.topic.split(':')[1]
const bookTicker: BookTicker = {
type: 'book_ticker',
symbol,
exchange: this._exchange,
askAmount: message.data.bestAskSize !== undefined && message.data.bestAskSize !== null ? Number(message.data.bestAskSize) : undefined,
askPrice: message.data.bestAsk !== undefined && message.data.bestAsk !== null ? Number(message.data.bestAsk) : undefined,
bidPrice: message.data.bestBid !== undefined && message.data.bestBid !== null ? Number(message.data.bestBid) : undefined,
bidAmount: message.data.bestBidSize !== undefined && message.data.bestBidSize !== null ? Number(message.data.bestBidSize) : undefined,
timestamp: new Date(message.data.time),
localTimestamp: localTimestamp
}
yield bookTicker
}
}
type KucoinTickerMessage = {
type: 'message'
topic: '/market/ticker:ADA-USDT'
subject: 'trade.ticker'
data: {
bestAsk: '0.549931'
bestAskSize: '966.4756'
bestBid: '0.549824'
bestBidSize: '1050'
price: '0.549825'
sequence: '1623526404099'
size: '1'
time: 1660608019871
}
}
type KucoinTradeMessage = {
type: 'message'
topic: '/market/match:BTC-USDT'
subject: 'trade.l3match'
data: {
symbol: 'BTC-USDT'
side: 'sell'
type: 'match'
makerOrderId: '62fadde41add68000167fb58'
sequence: '1636276321894'
size: '0.00001255'
price: '24093.9'
takerOrderId: '62faddfff0476c0001c86c71'
time: '1660608000026914990'
tradeId: '62fade002e113d292303a18b'
}
}
type LocalDepthInfo = {
bufferedUpdates: CircularBuffer
snapshotProcessed?: boolean
lastUpdateId?: number
validatedFirstUpdate?: boolean
}
type KucoinLevel2SnapshotMessage = {
type: 'message'
generated: true
topic: '/market/level2Snapshot:BTC-USDT'
subject: 'trade.l2Snapshot'
code: '200000'
data: {
time: 1660608003710
sequence: '1636276324355'
bids: [string, string][] | null
asks: [string, string][] | null
}
}
type KucoinLevel2UpdateMessage =
| {
type: 'message'
topic: '/market/level2:BTC-USDT'
subject: 'trade.l2update'
data: {
sequenceStart: 1636276324710
symbol: 'BTC-USDT'
changes: { asks: [string, string, string][]; bids: [string, string, string][] }
sequenceEnd: 1636276324710
time: undefined
}
}
| {
type: 'message'
topic: '/market/level2:BTC-USDT'
subject: 'trade.l2update'
data: {
changes: { asks: []; bids: [['27309.8', '0.35127929', '8005280396']] }
sequenceEnd: 8005280396
sequenceStart: 8005280396
symbol: 'BTC-USDT'
time: 1685578980002
}
}
================================================
FILE: src/mappers/kucoinfutures.ts
================================================
import { debug } from '../debug.ts'
import { asNumberIfValid, CircularBuffer, upperCaseSymbols } from '../handy.ts'
import { BookChange, BookTicker, DerivativeTicker, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
export class KucoinFuturesTradesMapper implements Mapper<'kucoin-futures', Trade> {
canHandle(message: KucoinFuturesTradeMessage) {
return message.type === 'message' && message.topic.startsWith('/contractMarket/execution')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'contractMarket/execution',
symbols
} as const
]
}
*map(message: KucoinFuturesTradeMessage, localTimestamp: Date): IterableIterator {
const kucoinTrade = message.data
const timestamp = new Date(kucoinTrade.ts / 1000000)
yield {
type: 'trade',
symbol: kucoinTrade.symbol,
exchange: 'kucoin-futures',
id: kucoinTrade.tradeId,
price: Number(kucoinTrade.price),
amount: Number(kucoinTrade.size),
side: kucoinTrade.side === 'sell' ? 'sell' : 'buy',
timestamp,
localTimestamp
}
}
}
export class KucoinFuturesBookChangeMapper implements Mapper<'kucoin-futures', BookChange> {
protected readonly symbolToDepthInfoMapping: {
[key: string]: LocalDepthInfo
} = {}
constructor(private readonly ignoreBookSnapshotOverlapError: boolean) {}
canHandle(message: KucoinFuturesLevel2SnapshotMessage | KucoinFuturesLevel2UpdateMessage) {
return message.type === 'message' && message.topic.startsWith('/contractMarket/level2')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'contractMarket/level2',
symbols
} as const,
{
channel: 'contractMarket/level2Snapshot',
symbols
} as const
]
}
*map(message: KucoinFuturesLevel2SnapshotMessage | KucoinFuturesLevel2UpdateMessage, localTimestamp: Date) {
const symbol = message.topic.split(':')[1]
if (this.symbolToDepthInfoMapping[symbol] === undefined) {
this.symbolToDepthInfoMapping[symbol] = {
bufferedUpdates: new CircularBuffer(2000)
}
}
const symbolDepthInfo = this.symbolToDepthInfoMapping[symbol]
const snapshotAlreadyProcessed = symbolDepthInfo.snapshotProcessed
// first check if received message is snapshot and process it as such if it is
if (message.subject === 'level2Snapshot') {
// if we've already received 'manual' snapshot, ignore if there is another one
if (snapshotAlreadyProcessed) {
return
}
// produce snapshot book_change
const kucoinSnapshotData = message.data
if (!message.data) {
return
}
if (!kucoinSnapshotData.asks) {
kucoinSnapshotData.asks = []
}
if (!kucoinSnapshotData.bids) {
kucoinSnapshotData.bids = []
}
// mark given symbol depth info that has snapshot processed
symbolDepthInfo.lastUpdateId = Number(kucoinSnapshotData.sequence)
symbolDepthInfo.snapshotProcessed = true
// if there were any depth updates buffered, let's process those by adding to or updating the initial snapshot
for (const update of symbolDepthInfo.bufferedUpdates.items()) {
const bookChange = this.mapBookDepthUpdate(update, localTimestamp)
if (bookChange !== undefined) {
const mappedChange = this.mapChange(update.data.change)
if (mappedChange.price == 0) {
continue
}
const matchingSide = mappedChange.isBid ? kucoinSnapshotData.bids : kucoinSnapshotData.asks
const matchingLevel = matchingSide.find((b) => b[0] === mappedChange.price)
if (matchingLevel !== undefined) {
// remove empty level from snapshot
if (mappedChange.amount === 0) {
const index = matchingSide.findIndex((b) => b[0] === mappedChange.price)
if (index > -1) {
matchingSide.splice(index, 1)
}
} else {
matchingLevel[1] = mappedChange.amount
}
} else if (mappedChange.amount != 0) {
matchingSide.push([mappedChange.price, mappedChange.amount])
}
}
}
// remove all buffered updates
symbolDepthInfo.bufferedUpdates.clear()
const bookChange: BookChange = {
type: 'book_change',
symbol,
exchange: 'kucoin-futures',
isSnapshot: true,
bids: kucoinSnapshotData.bids.map(this.mapBookLevel),
asks: kucoinSnapshotData.asks.map(this.mapBookLevel),
timestamp: localTimestamp,
localTimestamp
}
yield bookChange
} else if (snapshotAlreadyProcessed) {
// snapshot was already processed let's map the message as normal book_change
const bookChange = this.mapBookDepthUpdate(message, localTimestamp)
if (bookChange !== undefined) {
yield bookChange
}
} else {
symbolDepthInfo.bufferedUpdates.append(message)
}
}
protected mapBookDepthUpdate(l2UpdateMessage: KucoinFuturesLevel2UpdateMessage, localTimestamp: Date): BookChange | undefined {
// we can safely assume here that depthContext and lastUpdateId aren't null here as this is method only works
// when we've already processed the snapshot
const symbol = l2UpdateMessage.topic.split(':')[1]
const depthContext = this.symbolToDepthInfoMapping[symbol]!
const lastUpdateId = depthContext.lastUpdateId!
// Drop any event where sequence is <= lastUpdateId in the snapshot
if (l2UpdateMessage.data.sequence <= lastUpdateId) {
return
}
// The first processed event should have sequence>lastUpdateId
if (!depthContext.validatedFirstUpdate) {
// if there is new instrument added it can have empty book at first and that's normal
const bookSnapshotIsEmpty = lastUpdateId == -1 || lastUpdateId == 0
if (l2UpdateMessage.data.sequence === lastUpdateId + 1 || bookSnapshotIsEmpty) {
depthContext.validatedFirstUpdate = true
} else {
const message = `Book depth snapshot has no overlap with first update, update ${JSON.stringify(
l2UpdateMessage
)}, lastUpdateId: ${lastUpdateId}, exchange kucoin-futures`
if (this.ignoreBookSnapshotOverlapError) {
depthContext.validatedFirstUpdate = true
debug(message)
} else {
throw new Error(message)
}
}
}
const change = this.mapChange(l2UpdateMessage.data.change)
return {
type: 'book_change',
symbol: symbol,
exchange: 'kucoin-futures',
isSnapshot: false,
bids: change.isBid
? [
{
price: change.price,
amount: change.amount
}
]
: [],
asks:
change.isBid === false
? [
{
price: change.price,
amount: change.amount
}
]
: [],
timestamp: new Date(l2UpdateMessage.data.timestamp),
localTimestamp: localTimestamp
}
}
private mapBookLevel(level: [number, number]) {
return { price: level[0], amount: level[1] }
}
private mapChange(change: string) {
const parts = change.split(',')
const isBid = parts[1] === 'buy'
const price = Number(parts[0])
const amount = Number(parts[2])
return { isBid, price, amount }
}
}
export class KucoinFuturesBookTickerMapper implements Mapper<'kucoin-futures', BookTicker> {
canHandle(message: KucoinFuturesTickerMessage) {
return message.type === 'message' && message.topic.startsWith('/contractMarket/tickerV2')
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'contractMarket/tickerV2',
symbols
} as const
]
}
*map(message: KucoinFuturesTickerMessage, localTimestamp: Date) {
const symbol = message.topic.split(':')[1]
const bookTicker: BookTicker = {
type: 'book_ticker',
symbol,
exchange: 'kucoin-futures',
askAmount: message.data.bestAskSize !== undefined && message.data.bestAskSize !== null ? message.data.bestAskSize : undefined,
askPrice:
message.data.bestAskPrice !== undefined && message.data.bestAskPrice !== null ? Number(message.data.bestAskPrice) : undefined,
bidPrice:
message.data.bestBidPrice !== undefined && message.data.bestBidPrice !== null ? Number(message.data.bestBidPrice) : undefined,
bidAmount: message.data.bestBidSize !== undefined && message.data.bestBidSize !== null ? message.data.bestBidSize : undefined,
timestamp: new Date(message.data.ts / 1000000),
localTimestamp: localTimestamp
}
yield bookTicker
}
}
export class KucoinFuturesDerivativeTickerMapper implements Mapper<'kucoin-futures', DerivativeTicker> {
private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
private readonly _lastPrices = new Map()
private readonly _openInterests = new Map()
canHandle(message: KucoinFuturesTickerMessage) {
return (
message.type === 'message' &&
(message.topic.startsWith('/contract/instrument') ||
message.topic.startsWith('/contractMarket/execution') ||
message.topic.startsWith('/contract/details'))
)
}
getFilters(symbols?: string[]) {
symbols = upperCaseSymbols(symbols)
return [
{
channel: 'contract/instrument',
symbols
} as const,
{
channel: 'contractMarket/execution',
symbols
} as const,
{
channel: 'contract/details',
symbols
} as const
]
}
*map(message: KucoinFuturesInstrumentMessage | KucoinFuturesTradeMessage, localTimestamp: Date): IterableIterator