Repository: MutinyWallet/blastr Branch: master Commit: 99b4e364596f Files: 16 Total size: 66.2 KB Directory structure: gitextract_itwyxuif/ ├── .github/ │ └── workflows/ │ ├── publish-staging.yml │ ├── publish.yml │ └── test.yml ├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md ├── migrations/ │ ├── 0000_events.sql │ └── 0001_tag.sql ├── package.json ├── src/ │ ├── db.rs │ ├── error.rs │ ├── lib.rs │ ├── nostr.rs │ └── utils.rs └── wrangler.toml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/publish-staging.yml ================================================ name: Publish Staging on: workflow_dispatch: jobs: deploy: runs-on: ubuntu-latest name: Deploy steps: - uses: actions/checkout@master - name: Publish env: CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }} run: npm install -g wrangler && wrangler publish --env staging ================================================ FILE: .github/workflows/publish.yml ================================================ name: Publish on: push: branches: - master jobs: deploy: runs-on: ubuntu-latest name: Deploy steps: - uses: actions/checkout@master - name: Publish env: CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }} run: npm install -g wrangler && wrangler publish ================================================ FILE: .github/workflows/test.yml ================================================ name: Tests on: pull_request: jobs: website: name: Build WASM binary runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: toolchain: nightly components: clippy target: wasm32-unknown-unknown override: true profile: minimal - uses: jetli/wasm-pack-action@v0.4.0 with: version: 'latest' - uses: actions/cache@v2 with: path: | ~/.cargo/registry ~/.cargo/git target key: cargo-${{ runner.os }}-browser-tests-${{ hashFiles('**/Cargo.toml') }} restore-keys: | cargo-${{ runner.os }}-browser-tests- cargo-${{ runner.os }}- - name: Build wasm run: npm install -g wrangler && wrangler build rust_tests: name: Rust Checks runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: toolchain: nightly components: clippy, rustfmt target: wasm32-unknown-unknown override: true profile: minimal - name: Setup trunk uses: jetli/trunk-action@v0.1.0 with: version: 'latest' - uses: actions/cache@v2 with: path: | ~/.cargo/registry ~/.cargo/git target key: cargo-${{ runner.os }}-rust-tests-${{ hashFiles('**/Cargo.toml') }} restore-keys: | cargo-${{ runner.os }}-rust-tests- cargo-${{ runner.os }}- - name: Check formatting working-directory: . run: cargo fmt --check - name: Check clippy working-directory: . run: cargo clippy ================================================ FILE: .gitignore ================================================ # Generated by Cargo # will have compiled files and executables /target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk # cloudflare things .DS_Store /node_modules **/*.rs.bk wasm-pack.log build/ /target /dist .dev.vars .wrangler ================================================ FILE: Cargo.toml ================================================ [package] name = "blastr" version = "0.0.0" edition = "2018" resolver = "2" [lib] crate-type = ["cdylib", "rlib"] [features] default = ["console_error_panic_hook"] [dependencies] cfg-if = "0.1.2" worker = { version = "0.0.18", features = ["queue", "d1"] } futures = "0.3.26" futures-util = { version = "0.3", default-features = false } nostr = { version = "0.22.0", default-features = false, features = ["nip11"] } serde = { version = "^1.0", features = ["derive"] } serde_json = "1.0.67" # The `console_error_panic_hook` crate provides better debugging of panics by # logging them with `console.error`. This is great for development, but requires # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for # code size when deploying. console_error_panic_hook = { version = "0.1.1", optional = true } [profile.release] # Tell `rustc` to optimize for small code size. opt-level = "s" ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2023 Mutiny Wallet Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # blastr A nostr cloudflare workers proxy relay that publishes to all known online relays. ![blastr diagram](./docs/images/blastr-diagram.png) This takes advantage of the high availabilty of cloudflare serverless workers on the edge that are rust wasm-based with 0ms cold starts. Learn more about [cloudflare workers](https://workers.cloudflare.com/). This is write only for now and is compatible with nostr clients but also features a simple POST api endpoint at `/event`. All events get queued up to run in batches by another worker that spins up every 30s if there's any events lined up, or once a certain amount of events are queued up. This will help ensure that your events are broadcasted to as many places as possible. ## Development With `wrangler`, you can build, test, and deploy your Worker with the following commands: ```sh # install wrangler if you do not have it yet $ npm install -g wrangler # log into cloudflare if you havent before $ wrangler login # compiles your project to WebAssembly and will warn of any issues $ npm run build # run your Worker in an ideal development workflow (with a local server, file watcher & more) $ npm run dev # deploy your Worker globally to the Cloudflare network (update your wrangler.toml file for configuration) $ npm run deploy ``` ## Staging Production deployments happen automatically but staging is manually for now. Deploy whenever you need to test changes before merging to master. ``` wrangler publish --env staging ``` ### Setup There's a few cloudflare components that Blastr uses behind the scenes, namely a KV store and multiple queues to distribute the load. Right now some of these are hardcoded for us since they have to map from the `wrangler.toml` file to the rust codebase. Need a TODO for making this more dynamic. #### KV store This doesn't rebroadcast events that have already been broadcasted before. So we have a KV for that. We also have a KV for storing the NWC requests and responses to get around ephemeral events. ``` wrangler kv:namespace create PUBLISHED_NOTES wrangler kv:namespace create PUBLISHED_NOTES --preview wrangler kv:namespace create NWC_REQUESTS wrangler kv:namespace create NWC_REQUESTS --preview wrangler kv:namespace create NWC_RESPONSES wrangler kv:namespace create NWC_RESPONSES --preview ``` #### Queues ``` wrangler queues create nostr-events-pub-1-b wrangler queues create nostr-events-pub-2-b wrangler queues create nostr-events-pub-3-b wrangler queues create nostr-events-pub-4-b wrangler queues create nostr-events-pub-5-b wrangler queues create nostr-events-pub-6-b wrangler queues create nostr-events-pub-7-b wrangler queues create nostr-events-pub-8-b wrangler queues create nostr-events-pub-9-b wrangler queues create nostr-events-pub-10-b ``` Read the latest `worker` crate documentation here: https://docs.rs/worker ### CICD There's an example workflow here for publishing on master branch pushes. You need to set `CF_API_TOKEN` in your github repo secrets first. You also should either remove or configure `wrangler.toml` to point to a custom domain of yours: ``` routes = [ { pattern = "example.com/about", zone_id = "" } # replace with your info ] ``` and any other info in `wrangler.toml` that is custom to you, like the names / id's of queues or kv's. ### WebAssembly `workers-rs` (the Rust SDK for Cloudflare Workers used in this template) is meant to be executed as compiled WebAssembly, and as such so **must** all the code you write and depend upon. All crates and modules used in Rust-based Workers projects have to compile to the `wasm32-unknown-unknown` triple. Read more about this on the [`workers-rs`](https://github.com/cloudflare/workers-rs) project README. ================================================ FILE: migrations/0000_events.sql ================================================ -- Migration number: 0000 2023-08-10T16:43:44.275Z DROP TABLE IF EXISTS event; CREATE TABLE IF NOT EXISTS event ( id BLOB PRIMARY KEY, created_at INTEGER NOT NULL, pubkey BLOB NOT NULL, kind INTEGER NOT NULL, content TEXT NOT NULL, sig BLOB NOT NULL, deleted INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS pubkey_index ON event(pubkey); CREATE INDEX IF NOT EXISTS kind_index ON event(kind); ================================================ FILE: migrations/0001_tag.sql ================================================ -- Migration number: 0001 2023-08-11T20:22:11.351Z DROP TABLE IF EXISTS tag; CREATE TABLE IF NOT EXISTS tag ( id INTEGER PRIMARY KEY, event_id BLOB NOT NULL, name TEXT NOT NULL, value TEXT, FOREIGN KEY(event_id) REFERENCES event(id) ON DELETE CASCADE UNIQUE(event_id, name, value) ); CREATE INDEX IF NOT EXISTS tag_event_index ON tag(event_id); CREATE INDEX IF NOT EXISTS tag_name_index ON tag(name); CREATE INDEX IF NOT EXISTS tag_value_index ON tag(value); ================================================ FILE: package.json ================================================ { "private": true, "version": "0.0.0", "scripts": { "deploy": "wrangler publish", "dev": "wrangler dev --local" }, "devDependencies": { "@miniflare/tre": "^3.0.0-next.8", "wrangler": "^2.0.0" } } ================================================ FILE: src/db.rs ================================================ use std::collections::HashMap; use ::nostr::{Event, Kind, RelayMessage}; use nostr::{EventId, Tag, TagKind, Timestamp}; use serde::Deserialize; use serde_json::Value; use wasm_bindgen::prelude::JsValue; use worker::D1Database; use worker::*; #[derive(Debug, Deserialize)] struct EventRow { id: EventId, pubkey: nostr::secp256k1::XOnlyPublicKey, created_at: Timestamp, kind: Kind, tags: Option>, content: String, sig: nostr::secp256k1::schnorr::Signature, } #[derive(Deserialize)] struct TagRow { event_id: EventId, name: String, value: String, } pub async fn get_nwc_events(keys: &[String], kind: Kind, db: &D1Database) -> Result> { // Determine the event kind match kind { Kind::WalletConnectResponse => (), Kind::WalletConnectRequest => (), _ => return Ok(vec![]), // skip other event types }; console_log!("querying for ({keys:?}) and {}", kind.as_u32()); // Query for the events first, without the tags let placeholders: String = keys.iter().map(|_| "?").collect::>().join(", "); let query_str = format!( r#" SELECT * FROM event WHERE pubkey IN ({}) AND kind = ? AND deleted = 0 "#, placeholders ); let mut stmt = db.prepare(&query_str); let mut bindings = Vec::with_capacity(keys.len() + 1); // +1 for the kind afterwards for key in keys.iter() { bindings.push(JsValue::from_str(key)); } bindings.push(JsValue::from_f64(kind.as_u32() as f64)); stmt = stmt.bind(&bindings)?; let result = stmt.all().await.map_err(|e| { console_log!("Failed to fetch nwc events: {}", e); format!("Failed to fetch nwc events: {}", e) })?; let mut events: Vec = result .results::()? .iter() .map(|row| { let e: EventRow = serde_json::from_value(row.clone()).map_err(|e| { console_log!("failed to parse event: {}", e); worker::Error::from(format!( "Failed to deserialize event from row ({}): {}", row, e )) })?; Ok(Event { id: e.id, pubkey: e.pubkey, created_at: e.created_at, kind: e.kind, tags: e.tags.unwrap_or_default(), content: e.content, sig: e.sig, }) }) .collect::>>()?; // Now get all the tags for all the events found let event_ids: Vec = events.iter().map(|e| e.id).collect(); let placeholders: String = event_ids.iter().map(|_| "?").collect::>().join(", "); let tag_query_str = format!( r#" SELECT event_id, name, value FROM tag WHERE event_id IN ({}) ORDER BY id ASC "#, placeholders ); let mut tag_stmt = db.prepare(&tag_query_str); let bindings: Vec = event_ids .iter() .map(|id| JsValue::from_str(&id.to_string())) .collect(); tag_stmt = tag_stmt.bind(&bindings)?; let tag_result = tag_stmt .all() .await .map_err(|e| format!("Failed to fetch tags: {}", e))?; let tags: Vec = tag_result.results::()?; let mut tags_map: HashMap> = HashMap::new(); for tag_row in tags { if let Ok(tag) = Tag::parse(vec![tag_row.name, tag_row.value]) { tags_map .entry(tag_row.event_id) .or_insert_with(Vec::new) .push(tag); } } for event in &mut events { if let Some(tags) = tags_map.remove(&event.id) { event.tags.extend(tags); } } // Tag ordering could screw up signature, though it shouldn't matter // for NWC messages because those should only have one tag. // Also we insert tags in order and do an ORDER BY id so it should be fine. events.retain(|event| match event.verify() { Ok(_) => true, Err(e) => { console_log!("Verification failed for event with id {}: {}", event.id, e); false } }); Ok(events) } pub async fn handle_nwc_event(event: Event, db: &D1Database) -> Result> { // Determine the event kind match event.kind { Kind::WalletConnectResponse => (), Kind::WalletConnectRequest => (), _ => return Ok(None), // skip other event types }; // Create the main event insertion query. let event_insert_query = worker::query!( db, r#" INSERT OR IGNORE INTO event (id, created_at, pubkey, kind, content, sig) VALUES (?, ?, ?, ?, ?, ?) "#, &event.id, &event.created_at, &event.pubkey, &event.kind, &event.content, &event.sig )?; // Create a vector of tag insertion queries. let mut tag_insert_queries: Vec<_> = event .tags .iter() .map(|tag| { worker::query!( db, r#" INSERT OR IGNORE INTO tag (event_id, name, value) VALUES (?, ?, ?) "#, &event.id, &tag.kind().to_string(), &tag.as_vec().get(1) ) .expect("should compile query") }) .collect(); // Combine the main event and tag insertion queries. let mut batch_queries = vec![event_insert_query]; batch_queries.append(&mut tag_insert_queries); // Run the batch queries. let mut results = db.batch(batch_queries).await?.into_iter(); // Check the result of the main event insertion. if let Some(error_msg) = results.next().and_then(|res| res.error()) { console_log!("error saving nwc event to event table: {}", error_msg); let relay_msg = RelayMessage::new_ok(event.id, false, &error_msg); return Ok(Some(relay_msg)); } // Check the results for the tag insertions. for tag_insert_result in results { if let Some(error_msg) = tag_insert_result.error() { console_log!("error saving tag to tag table: {}", error_msg); let relay_msg = RelayMessage::new_ok(event.id, false, &error_msg); return Ok(Some(relay_msg)); } } let relay_msg = RelayMessage::new_ok(event.id, true, ""); Ok(Some(relay_msg)) } /// When a NWC request has been fulfilled, soft delete the request from the database pub async fn delete_nwc_request(event: Event, db: &D1Database) -> Result<()> { // Filter only relevant events match event.kind { Kind::WalletConnectResponse => (), _ => return Ok(()), // skip other event types }; let p_tag = event.tags.iter().find(|t| t.kind() == TagKind::P).cloned(); let e_tag = event.tags.iter().find(|t| t.kind() == TagKind::E).cloned(); if let Some(Tag::PubKey(pubkey, ..)) = p_tag { if let Some(Tag::Event(event_id, ..)) = e_tag { // Soft delete the event based on pubkey and event_id match worker::query!( db, "UPDATE event SET deleted = 1 WHERE pubkey = ? AND id = ?", &pubkey.to_string(), &event_id )? .run() .await { Ok(_) => (), Err(e) => { console_log!("error soft deleting nwc event from database: {e}"); return Ok(()); } } } } Ok(()) } /// When a NWC response has been fulfilled, soft delete the response from the database pub async fn delete_nwc_response(event: &Event, db: &D1Database) -> Result<()> { // Filter only relevant events match event.kind { Kind::WalletConnectResponse => (), _ => return Ok(()), // skip other event types }; // Soft delete the event based on pubkey and id match worker::query!( db, "UPDATE event SET deleted = 1 WHERE pubkey = ? AND id = ?", &event.pubkey.to_string(), &event.id )? .run() .await { Ok(_) => console_log!("soft deleted nwc response event: {}", event.id), Err(e) => { console_log!("error soft deleting nwc event from database: {e}"); return Ok(()); } } Ok(()) } ================================================ FILE: src/error.rs ================================================ pub enum Error { /// Worker error WorkerError(String), } impl From for Error { fn from(e: worker::Error) -> Self { Error::WorkerError(e.to_string()) } } ================================================ FILE: src/lib.rs ================================================ use crate::nostr::NOSTR_QUEUE_8; use crate::nostr::NOSTR_QUEUE_9; pub(crate) use crate::nostr::{ try_queue_event, NOSTR_QUEUE, NOSTR_QUEUE_2, NOSTR_QUEUE_3, NOSTR_QUEUE_4, NOSTR_QUEUE_5, NOSTR_QUEUE_6, }; use crate::{db::delete_nwc_request, nostr::NOSTR_QUEUE_10}; use crate::{db::get_nwc_events, nostr::NOSTR_QUEUE_7}; use crate::{db::handle_nwc_event, nostr::get_nip11_response}; use ::nostr::{ClientMessage, Event, EventId, Filter, Kind, RelayMessage, SubscriptionId, Tag}; use futures::StreamExt; use futures_util::lock::Mutex; use serde::{Deserialize, Serialize}; use std::ops::DerefMut; use std::rc::Rc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use worker::*; mod db; mod error; mod nostr; mod utils; fn log_request(req: &Request) { console_log!( "Incoming Request: {} - [{}]", Date::now().to_string(), req.path(), ); } #[derive(Serialize, Deserialize, Clone)] pub struct PublishedNote { date: String, } /// The list of event kinds that are disallowed for the Nostr relay /// currently, this is just the NIP-95 event const DISALLOWED_EVENT_KINDS: [u32; 1] = [1064]; /// Main function for the Cloudflare Worker that triggers off of a HTTP req #[event(fetch)] pub async fn main(req: Request, env: Env, _ctx: Context) -> Result { log_request(&req); // Optionally, get more helpful error messages written to the console in the case of a panic. utils::set_panic_hook(); // Optionally, use the Router to handle matching endpoints, use ":name" placeholders, or "*name" // catch-alls to match on specific patterns. Alternatively, use `Router::with_data(D)` to // provide arbitrary data that will be accessible in each route via the `ctx.data()` method. let router = Router::new(); // Add as many routes as your Worker needs! Each route will get a `Request` for handling HTTP // functionality and a `RouteContext` which you can use to and get route parameters and // Environment bindings like KV Stores, Durable Objects, Secrets, and Variables. router .post_async("/event", |mut req, ctx| async move { // for any adhoc POST event match req.text().await { Ok(request_text) => { if let Ok(client_msg) = ClientMessage::from_json(request_text) { match client_msg { ClientMessage::Event(event) => { console_log!("got an event from client: {}", event.id); match event.verify() { Ok(()) => (), Err(e) => { console_log!("could not verify event {}: {}", event.id, e); let relay_msg = RelayMessage::new_ok(event.id, false, "invalid event"); return relay_response(relay_msg); } } // check if disallowed event kind if DISALLOWED_EVENT_KINDS.contains(&event.kind.as_u32()) { console_log!( "invalid event kind {}: {}", event.kind.as_u32(), event.id ); let relay_msg = RelayMessage::new_ok( event.id, false, "disallowed event kind", ); return relay_response(relay_msg); }; // check if we've already published it before let published_notes = ctx.kv("PUBLISHED_NOTES")?; if published_notes .get(event.id.to_string().as_str()) .json::() .await .ok() .flatten() .is_some() { console_log!("event already published: {}", event.id); let relay_msg = RelayMessage::new_ok( event.id, false, "event already published", ); return relay_response(relay_msg); }; let db = ctx.d1("DB")?; // if the event is a nostr wallet connect event, we // should save it and not send to other relays. if let Some(relay_msg) = handle_nwc_event(*event.clone(), &db).await? { if let Err(e) = delete_nwc_request(*event, &db).await { console_log!("failed to delete nwc request: {e}"); } return relay_response(relay_msg); }; // broadcast it to all queues let nostr_queues = vec![ ctx.env.queue(NOSTR_QUEUE).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_2).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_3).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_4).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_5).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_6).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_7).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_8).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_9).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_10).expect("get queue"), ]; try_queue_event(*event.clone(), nostr_queues).await; console_log!("queued up nostr event: {}", event.id); match published_notes .put( event.id.to_string().as_str(), PublishedNote { date: Date::now().to_string(), }, )? .execute() .await { Ok(_) => { console_log!("saved published note: {}", event.id); let relay_msg = RelayMessage::new_ok(event.id, true, ""); relay_response(relay_msg) } Err(e) => { console_log!( "could not save published note: {} - {e:?}", event.id ); let relay_msg = RelayMessage::new_ok( event.id, false, "error: could not save published note", ); relay_response(relay_msg) } } } _ => { console_log!("ignoring other nostr client message types"); Response::error("Only Event types allowed", 400) } } } else { Response::error("Could not parse Client Message", 400) } } Err(e) => { console_log!("could not get request text: {}", e); Response::error("Could not get request text", 400) } } }) .get("/", |req, ctx| { // NIP 11 if req.headers().get("Accept").ok().flatten() == Some("application/nostr+json".to_string()) { return Response::from_json(&get_nip11_response())?.with_cors(&cors()); } let ctx = Rc::new(ctx); // For websocket compatibility let pair = WebSocketPair::new()?; let server = pair.server; server.accept()?; console_log!("accepted websocket, about to spawn event stream"); wasm_bindgen_futures::spawn_local(async move { let running_thread = Arc::new(AtomicBool::new(false)); let new_subscription_req = Arc::new(AtomicBool::new(false)); let requested_filters = Arc::new(Mutex::new(Filter::new())); let mut event_stream = server.events().expect("stream error"); console_log!("spawned event stream, waiting for first message.."); while let Some(event) = event_stream.next().await { if let Err(e) = event { console_log!("error parsing some event: {e}"); continue; } match event.expect("received error in websocket") { WebsocketEvent::Message(msg) => { if msg.text().is_none() { continue; }; if let Ok(client_msg) = ClientMessage::from_json(msg.text().unwrap()) { match client_msg { ClientMessage::Event(event) => { console_log!("got an event from client: {}", event.id); match event.verify() { Ok(()) => (), Err(e) => { console_log!( "could not verify event {}: {}", event.id, e ); let relay_msg = RelayMessage::new_ok( event.id, false, "disallowed event kind", ); server .send_with_str(&relay_msg.as_json()) .expect("failed to send response"); continue; } } // check if disallowed event kind if DISALLOWED_EVENT_KINDS.contains(&event.kind.as_u32()) { console_log!( "invalid event kind {}: {}", event.kind.as_u32(), event.id ); let relay_msg = RelayMessage::new_ok( event.id, false, "disallowed event kind", ); server .send_with_str(&relay_msg.as_json()) .expect("failed to send response"); continue; }; // check if we've already published it before let published_notes = ctx.kv("PUBLISHED_NOTES").expect("get kv"); if published_notes .get(event.id.to_string().as_str()) .json::() .await .ok() .flatten() .is_some() { console_log!("event already published: {}", event.id); let relay_msg = RelayMessage::new_ok( event.id, false, "event already published", ); server .send_with_str(&relay_msg.as_json()) .expect("failed to send response"); continue; }; let db = ctx.d1("DB").expect("should have DB"); // if the event is a nostr wallet connect event, we // should save it and not send to other relays. if let Some(response) = handle_nwc_event(*event.clone(), &db) .await .expect("failed to handle nwc event") { server .send_with_str(&response.as_json()) .expect("failed to send response"); if let Err(e) = delete_nwc_request(*event, &db).await { console_log!("failed to delete nwc request: {e}"); } continue; }; // broadcast it to all queues let nostr_queues = vec![ ctx.env.queue(NOSTR_QUEUE).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_2).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_3).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_4).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_5).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_6).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_7).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_8).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_9).expect("get queue"), ctx.env.queue(NOSTR_QUEUE_10).expect("get queue"), ]; try_queue_event(*event.clone(), nostr_queues).await; console_log!("queued up nostr event: {}", event.id); match published_notes .put( event.id.to_string().as_str(), PublishedNote { date: Date::now().to_string(), }, ) .expect("saved note") .execute() .await { Ok(_) => { console_log!("saved published note: {}", event.id); } Err(e) => { console_log!( "could not save published note: {} - {e:?}", event.id ); } } let relay_msg = RelayMessage::new_ok(event.id, true, ""); server .send_with_str(&relay_msg.as_json()) .expect("failed to send response"); } ClientMessage::Req { subscription_id, filters, } => { new_subscription_req.swap(true, Ordering::Relaxed); console_log!( "got a new client request sub: {}, len: {}", subscription_id, filters.len() ); // for each filter we handle it every 10 seconds // by reading storage and sending any new events // one caveat is that this will send events multiple // times if they are in multiple filters let mut valid = false; for filter in filters { let valid_nwc = { let nwc_kinds = filter .kinds .as_ref() .map(|k| { k.contains(&Kind::WalletConnectResponse) || k.contains( &Kind::WalletConnectRequest, ) }) .unwrap_or(false); let has_authors = filter .authors .as_ref() .map(|a| !a.is_empty()) .unwrap_or(false); let has_pks = filter .pubkeys .as_ref() .map(|a| !a.is_empty()) .unwrap_or(false); nwc_kinds && (has_authors || has_pks) }; if valid_nwc { let mut master_guard = requested_filters.lock().await; let master_filter = master_guard.deref_mut(); // now add the new filters to the main filter // object. This is a bit of a hack but we only // check certain sub filters for NWC. combine_filters(master_filter, &filter); console_log!( "New filter count: {}", master_filter .pubkeys .as_ref() .map_or(0, Vec::len) ); valid = true; } } // only spin up a new one if there's not a // spawn_local already going with filters // when other filters are added in, it should // be picked up in the master filter let mut sent_event_count = 0; if !running_thread.load(Ordering::Relaxed) && valid { // set running thread to true running_thread.swap(true, Ordering::Relaxed); let db = ctx.d1("DB").expect("should have DB"); let sub_id = subscription_id.clone(); let server_clone = server.clone(); let master_clone = requested_filters.clone(); let new_subscription_req_clone = new_subscription_req.clone(); wasm_bindgen_futures::spawn_local(async move { let mut sent_events = vec![]; loop { let master = master_clone.lock().await; console_log!( "Checking filters: {}", master.pubkeys.as_ref().map_or(0, Vec::len) ); match handle_filter( &sent_events, sub_id.clone(), master.clone(), &server_clone, &db, ) .await { Ok(new_event_ids) => { // add new events to sent events sent_events.extend(new_event_ids); // send EOSE if necessary if new_subscription_req_clone .load(Ordering::Relaxed) || sent_event_count != sent_events.len() { let relay_msg = RelayMessage::new_eose( sub_id.clone(), ); server_clone .send_with_str( relay_msg.as_json(), ) .expect( "failed to send response", ); sent_event_count = sent_events.len(); new_subscription_req_clone .swap(false, Ordering::Relaxed); } } Err(e) => console_log!( "error handling filter: {e}" ), } drop(master); utils::delay(5_000).await; } }); } else if !valid { // if not a nwc filter, we just send EOSE let relay_msg = RelayMessage::new_eose(subscription_id); server .send_with_str(relay_msg.as_json()) .expect("failed to send response"); } } _ => { console_log!("ignoring other nostr client message types"); } } } } WebsocketEvent::Close(_) => { console_log!("closing"); break; } } } }); Response::from_websocket(pair.client) }) .get("/favicon.ico", |_, _| { let bytes: Vec = include_bytes!("../static/favicon.ico").to_vec(); Response::from_bytes(bytes)?.with_cors(&cors()) }) .options("/*catchall", |_, _| empty_response()) .run(req, env) .await } /// Main function for the Cloudflare Worker that triggers off the nostr event queue #[event(queue)] pub async fn main(message_batch: MessageBatch, _env: Env, _ctx: Context) -> Result<()> { // Deserialize the message batch let messages: Vec> = message_batch.messages()?; let mut events: Vec = messages.iter().map(|m| m.body.clone()).collect(); events.sort(); events.dedup(); let part = queue_number(message_batch.queue().as_str())?; match nostr::send_nostr_events(events, part).await { Ok(event_ids) => { for event_id in event_ids { console_log!("Sent nostr event: {}", event_id) } } Err(error::Error::WorkerError(e)) => { console_log!("worker error: {e}"); } } Ok(()) } pub fn queue_number(batch_name: &str) -> Result { match batch_name { NOSTR_QUEUE => Ok(0), NOSTR_QUEUE_2 => Ok(1), NOSTR_QUEUE_3 => Ok(2), NOSTR_QUEUE_4 => Ok(3), NOSTR_QUEUE_5 => Ok(4), NOSTR_QUEUE_6 => Ok(5), NOSTR_QUEUE_7 => Ok(6), NOSTR_QUEUE_8 => Ok(7), NOSTR_QUEUE_9 => Ok(8), NOSTR_QUEUE_10 => Ok(9), _ => Err("unexpected queue".into()), } } /// if the user requests a NWC event, we have those stored, /// we should send them to the user pub async fn handle_filter( sent_events: &[EventId], subscription_id: SubscriptionId, filter: Filter, server: &WebSocket, db: &D1Database, ) -> Result> { let mut events = vec![]; // get all authors and pubkeys let mut keys = filter.authors.unwrap_or_default(); keys.extend( filter .pubkeys .unwrap_or_default() .into_iter() .map(|p| p.to_string()), ); if filter .kinds .clone() .unwrap_or_default() .contains(&Kind::WalletConnectRequest) { let mut found_events = get_nwc_events(&keys, Kind::WalletConnectRequest, db) .await .unwrap_or_default(); // filter out events that have already been sent found_events.retain(|e| !sent_events.contains(&e.id)); events.extend(found_events); } if filter .kinds .unwrap_or_default() .contains(&Kind::WalletConnectResponse) { let mut found_events = get_nwc_events(&keys, Kind::WalletConnectResponse, db) .await .unwrap_or_default(); // filter out events that have already been sent found_events.retain(|e| !sent_events.contains(&e.id)); events.extend(found_events); } // if the filter is only requesting replies to a certain event filter to only those if let Some(event_ids) = filter.events { events.retain(|event| { event .tags .iter() .any(|t| matches!(t, Tag::Event(id, _, _) if event_ids.contains(id))) }) } if events.is_empty() { return Ok(vec![]); } if !events.is_empty() { // sort and dedup events events.sort_by(|a, b| a.created_at.cmp(&b.created_at)); events.dedup(); // send all found events to the user for event in events.clone() { console_log!("sending event to client: {}", &event.id); let relay_msg = RelayMessage::new_event(subscription_id.clone(), event); server .send_with_str(&relay_msg.as_json()) .expect("failed to send response"); } } let sent_event_ids: Vec = events.into_iter().map(|e| e.id).collect(); Ok(sent_event_ids) } // Helper function to extend a vector without duplicates fn extend_without_duplicates(master: &mut Vec, new: &Vec) { for item in new { if !master.contains(item) { master.push(item.clone()); } } } fn combine_filters(master_filter: &mut Filter, new_filter: &Filter) { // Check and extend for IDs if let Some(vec) = &new_filter.ids { let master_vec = master_filter.ids.get_or_insert_with(Vec::new); extend_without_duplicates(master_vec, vec); } // Check and extend for authors if let Some(vec) = &new_filter.authors { let master_vec = master_filter.authors.get_or_insert_with(Vec::new); extend_without_duplicates(master_vec, vec); } // Check and extend for kinds if let Some(vec) = &new_filter.kinds { let master_vec = master_filter.kinds.get_or_insert_with(Vec::new); extend_without_duplicates(master_vec, vec); } // Check and extend for events if let Some(vec) = &new_filter.events { let master_vec = master_filter.events.get_or_insert_with(Vec::new); extend_without_duplicates(master_vec, vec); } // Check and extend for pubkeys if let Some(vec) = &new_filter.pubkeys { let master_vec = master_filter.pubkeys.get_or_insert_with(Vec::new); extend_without_duplicates(master_vec, vec); } // Check and extend for hashtags if let Some(vec) = &new_filter.hashtags { let master_vec = master_filter.hashtags.get_or_insert_with(Vec::new); extend_without_duplicates(master_vec, vec); } // Check and extend for references if let Some(vec) = &new_filter.references { let master_vec = master_filter.references.get_or_insert_with(Vec::new); extend_without_duplicates(master_vec, vec); } // Check and extend for identifiers if let Some(vec) = &new_filter.identifiers { let master_vec = master_filter.identifiers.get_or_insert_with(Vec::new); extend_without_duplicates(master_vec, vec); } } fn relay_response(msg: RelayMessage) -> worker::Result { Response::from_json(&msg)?.with_cors(&cors()) } fn empty_response() -> worker::Result { Response::empty()?.with_cors(&cors()) } fn cors() -> Cors { Cors::new() .with_credentials(true) .with_origins(vec!["*"]) .with_allowed_headers(vec!["Content-Type"]) .with_methods(Method::all()) } ================================================ FILE: src/nostr.rs ================================================ use crate::error::Error; use crate::utils::delay; use futures::future::Either; use futures::pin_mut; use futures::StreamExt; use nostr::prelude::*; use std::string::ToString; use std::vec; use worker::WebsocketEvent; use worker::{console_log, Cache, Fetch, Queue, Response, WebSocket}; pub(crate) const NOSTR_QUEUE: &str = "nostr-events-pub-1-b"; pub(crate) const NOSTR_QUEUE_2: &str = "nostr-events-pub-2-b"; pub(crate) const NOSTR_QUEUE_3: &str = "nostr-events-pub-3-b"; pub(crate) const NOSTR_QUEUE_4: &str = "nostr-events-pub-4-b"; pub(crate) const NOSTR_QUEUE_5: &str = "nostr-events-pub-5-b"; pub(crate) const NOSTR_QUEUE_6: &str = "nostr-events-pub-6-b"; pub(crate) const NOSTR_QUEUE_7: &str = "nostr-events-pub-7-b"; pub(crate) const NOSTR_QUEUE_8: &str = "nostr-events-pub-8-b"; pub(crate) const NOSTR_QUEUE_9: &str = "nostr-events-pub-9-b"; pub(crate) const NOSTR_QUEUE_10: &str = "nostr-events-pub-10-b"; const RELAY_LIST_URL: &str = "https://api.nostr.watch/v1/online"; const RELAYS: [&str; 8] = [ "wss://nostr.zebedee.cloud", "wss://relay.snort.social", "wss://eden.nostr.land", "wss://nos.lol", "wss://brb.io", "wss://nostr.fmt.wiz.biz", "wss://relay.damus.io", "wss://nostr.wine", ]; pub fn get_nip11_response() -> RelayInformationDocument { let version = env!("CARGO_PKG_VERSION"); let supported_nips = vec![1, 11, 15, 20]; RelayInformationDocument { name: Some("Mutiny blastr relay".to_string()), description: Some("Mutiny blastr relay".to_string()), pubkey: Some( "df173277182f3155d37b330211ba1de4a81500c02d195e964f91be774ec96708".to_string(), ), contact: Some("team@mutinywallet.com".to_string()), supported_nips: Some(supported_nips), software: Some("git+https://github.com/MutinyWallet/blastr.git".to_string()), version: Some(version.to_string()), } } pub async fn try_queue_event(event: Event, nostr_queues: Vec) { for nostr_queue in nostr_queues.iter() { match queue_nostr_event_with_queue(nostr_queue, event.clone()).await { Ok(_) => {} Err(Error::WorkerError(e)) => { console_log!("worker error: {e}"); } } } } pub async fn queue_nostr_event_with_queue(nostr_queue: &Queue, event: Event) -> Result<(), Error> { nostr_queue.send(&event).await?; Ok(()) } async fn send_event_to_relay(messages: Vec, relay: &str) -> Result<(), Error> { // skip self if relay == "wss://nostr.mutinywallet.com" { return Ok(()); } let connect_timeout = delay(10_000); let connect_future = WebSocket::connect(relay.parse().unwrap()); pin_mut!(connect_timeout); pin_mut!(connect_future); let either = futures::future::select(connect_future, connect_timeout).await; let ws_opt = match either { Either::Left((res, _)) => res, Either::Right(_) => { console_log!("time delay hit, stopping..."); Err(worker::Error::RustError("Connection timeout".to_string())) } }; match ws_opt { Ok(ws) => { // It's important that we call this before we send our first message, otherwise we will // not have any event listeners on the socket to receive the echoed message. let event_stream = ws.events(); if let Some(e) = event_stream.as_ref().err() { console_log!("Error calling ws events from relay {relay}: {e:?}"); return Err(Error::WorkerError(String::from("WS connection error"))); } let mut event_stream = event_stream.unwrap(); if let Some(e) = ws.accept().err() { console_log!("Error accepting ws from relay {relay}: {e:?}"); return Err(e.into()); } for message in messages { if let Some(e) = ws.send_with_str(message.as_json()).err() { console_log!("Error sending event to relay {relay}: {e:?}") } } // log the first message we received from the relay let sleep = delay(1_000); let event_stream_fut = event_stream.next(); pin_mut!(event_stream_fut); pin_mut!(sleep); match futures::future::select(event_stream_fut, sleep).await { Either::Left((event_stream_res, _)) => { if let Some(event) = event_stream_res { let event = event?; if let WebsocketEvent::Message(msg) = event { if let Some(text) = msg.text() { console_log!("websocket event from relay {relay}: {text}") } } } } Either::Right(_) => { console_log!("time delay hit, stopping..."); // Sleep triggered before we got a websocket response } } if let Some(_e) = ws.close::(None, None).err() { console_log!("Error websocket to relay {relay}") } } Err(e) => { console_log!("Error connecting to relay {relay}: {e:?}"); return Err(Error::WorkerError(String::from("WS connection error"))); } }; Ok(()) } pub async fn send_nostr_events(events: Vec, part: u32) -> Result, Error> { let messages: Vec = events .iter() .map(|e| ClientMessage::new_event(e.clone())) .collect(); // pull in the relays from nostr watch list let cache = Cache::default(); let relays = if let Some(mut resp) = cache.get(RELAY_LIST_URL, true).await? { console_log!("cache hit for relays"); match resp.json::>().await { Ok(r) => r, Err(_) => RELAYS.iter().map(|x| x.to_string()).collect(), } } else { console_log!("no cache hit for relays"); match Fetch::Url(RELAY_LIST_URL.parse().unwrap()).send().await { Ok(mut nostr_resp) => { console_log!("retrieved online relay list"); match nostr_resp.json::>().await { Ok(r) => { let mut resp = Response::from_json(&r)?; // Cache API respects Cache-Control headers. Setting s-max-age to 10 // will limit the response to be in cache for 10 seconds max resp.headers_mut().set("cache-control", "s-maxage=1800")?; cache.put(RELAY_LIST_URL, resp.cloned()?).await?; match resp.json::>().await { Ok(r) => r, Err(e) => { console_log!("could not parse nostr relay list json: {}", e); RELAYS.iter().map(|x| x.to_string()).collect() } } } Err(e) => { console_log!("could not parse nostr relay list response: {}", e); RELAYS.iter().map(|x| x.to_string()).collect() } } } Err(e) => { console_log!("could not retrieve relay list: {}", e); RELAYS.iter().map(|x| x.to_string()).collect() } } }; // find range of elements for this part let sub_relays = get_sub_vec_range(relays, find_range_from_part(part)); let mut futures = Vec::new(); for relay in sub_relays.iter() { let fut = send_event_to_relay(messages.clone(), relay); futures.push(fut); } let combined_futures = futures::future::join_all(futures); let sleep = delay(120_000); pin_mut!(combined_futures); pin_mut!(sleep); futures::future::select(combined_futures, sleep).await; Ok(events.iter().map(|e| e.id).collect()) } fn get_sub_vec_range(original: Vec, range: (usize, usize)) -> Vec { let len = original.len(); if range.0 >= len { return vec![]; } let end = if range.1 >= len { len - 1 } else { range.1 }; original[range.0..end].to_vec() } fn find_range_from_part(part: u32) -> (usize, usize) { let start = 30 * part; let end = start + 29; (start as usize, end as usize) } ================================================ FILE: src/utils.rs ================================================ use cfg_if::cfg_if; use std::time::Duration; use worker::Delay; cfg_if! { // https://github.com/rustwasm/console_error_panic_hook#readme if #[cfg(feature = "console_error_panic_hook")] { extern crate console_error_panic_hook; pub use self::console_error_panic_hook::set_once as set_panic_hook; } else { #[inline] pub fn set_panic_hook() {} } } pub async fn delay(delay: u64) { let delay: Delay = Duration::from_millis(delay).into(); delay.await; } ================================================ FILE: wrangler.toml ================================================ name = "blastr" main = "build/worker/shim.mjs" compatibility_date = "2022-01-20" # replace with your domain info - TODO this might not be required but we added it for ours. routes = [ { pattern = "nostr.mutinywallet.com/", zone_id = "2b9268714ce8d1c4431e8046d4ba55d3" }, { pattern = "nostr.mutinywallet.com/event", zone_id = "2b9268714ce8d1c4431e8046d4ba55d3" } ] # replace with your KV store info # create the queues with `wrangler kv:namespace create PUBLISHED_NOTES` and the same command with the `--preview` flag. # put your queue IDs below kv_namespaces = [ { binding = "PUBLISHED_NOTES", id = "afa24a392a5a41f6b1655507dfd9b97a", preview_id = "0b334aece8d74c3ab90e3e99db569ce8" }, { binding = "NWC_REQUESTS", id = "e5bd788ddc16410bb108df0f2ae89e62", preview_id = "63d99f551a464ff78bbbbee7113cb658" }, { binding = "NWC_RESPONSES", id = "5b434d5eced84abaad1c9a44448ac71c", preview_id = "af27b55b58754562b4250dcd3682547b" }, ] [env.staging] name = "blastr-staging" routes = [ { pattern = "nostr-staging.mutinywallet.com/", zone_id = "2b9268714ce8d1c4431e8046d4ba55d3" }, { pattern = "nostr-staging.mutinywallet.com/event", zone_id = "2b9268714ce8d1c4431e8046d4ba55d3" } ] kv_namespaces = [ { binding = "PUBLISHED_NOTES", id = "afa24a392a5a41f6b1655507dfd9b97a", preview_id = "0b334aece8d74c3ab90e3e99db569ce8" }, { binding = "NWC_REQUESTS", id = "e5bd788ddc16410bb108df0f2ae89e62", preview_id = "63d99f551a464ff78bbbbee7113cb658" }, { binding = "NWC_RESPONSES", id = "5b434d5eced84abaad1c9a44448ac71c", preview_id = "af27b55b58754562b4250dcd3682547b" }, ] [[d1_databases]] binding = "DB" database_name = "blastr-db" database_id = "82736a30-841d-4c24-87d8-788763dacb01" [[env.staging.d1_databases]] binding = "DB" database_name = "blastr-db-staging" database_id = "ba4c227b-edf2-46a4-99fa-4b7bfa036800" [env.staging.vars] WORKERS_RS_VERSION = "0.0.18" ENVIRONMENT = "staging" [vars] WORKERS_RS_VERSION = "0.0.18" ENVIRONMENT = "production" # Replace with all the queues you created, if you named them different. # create the queues with: `wrangler queues create {NAME}` # TODO make these more dynamic [[queues.producers]] queue = "nostr-events-pub-1-b" binding = "nostr-events-pub-1-b" [[queues.producers]] queue = "nostr-events-pub-2-b" binding = "nostr-events-pub-2-b" [[queues.producers]] queue = "nostr-events-pub-3-b" binding = "nostr-events-pub-3-b" [[queues.producers]] queue = "nostr-events-pub-4-b" binding = "nostr-events-pub-4-b" [[queues.producers]] queue = "nostr-events-pub-5-b" binding = "nostr-events-pub-5-b" [[queues.producers]] queue = "nostr-events-pub-6-b" binding = "nostr-events-pub-6-b" [[queues.producers]] queue = "nostr-events-pub-7-b" binding = "nostr-events-pub-7-b" [[queues.producers]] queue = "nostr-events-pub-8-b" binding = "nostr-events-pub-8-b" [[queues.producers]] queue = "nostr-events-pub-9-b" binding = "nostr-events-pub-9-b" [[queues.producers]] queue = "nostr-events-pub-10-b" binding = "nostr-events-pub-10-b" # consumers [[queues.consumers]] queue = "nostr-events-pub-1-b" max_batch_size = 100 max_batch_timeout = 5 # this is the best one, run quicker [[queues.consumers]] queue = "nostr-events-pub-2-b" max_batch_size = 100 max_batch_timeout = 15 [[queues.consumers]] queue = "nostr-events-pub-3-b" max_batch_size = 100 max_batch_timeout = 15 [[queues.consumers]] queue = "nostr-events-pub-4-b" max_batch_size = 100 max_batch_timeout = 15 [[queues.consumers]] queue = "nostr-events-pub-5-b" max_batch_size = 100 max_batch_timeout = 15 [[queues.consumers]] queue = "nostr-events-pub-6-b" max_batch_size = 100 max_batch_timeout = 15 [[queues.consumers]] queue = "nostr-events-pub-7-b" max_batch_size = 100 max_batch_timeout = 15 [[queues.consumers]] queue = "nostr-events-pub-8-b" max_batch_size = 100 max_batch_timeout = 15 [[queues.consumers]] queue = "nostr-events-pub-9-b" max_batch_size = 100 max_batch_timeout = 15 [[queues.consumers]] queue = "nostr-events-pub-10-b" max_batch_size = 100 max_batch_timeout = 15 [[env.staging.queues.producers]] queue = "nostr-events-pub-1-b" binding = "nostr-events-pub-1-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-2-b" binding = "nostr-events-pub-2-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-3-b" binding = "nostr-events-pub-3-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-4-b" binding = "nostr-events-pub-4-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-5-b" binding = "nostr-events-pub-5-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-6-b" binding = "nostr-events-pub-6-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-7-b" binding = "nostr-events-pub-7-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-8-b" binding = "nostr-events-pub-8-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-9-b" binding = "nostr-events-pub-9-b" [[env.staging.queues.producers]] queue = "nostr-events-pub-10-b" binding = "nostr-events-pub-10-b" [build] command = "cargo install -q worker-build --version 0.0.10 && worker-build --release"