Repository: digital-fabric/tipi
Branch: master
Commit: 7fd15c92e7dd
Files: 99
Total size: 173.1 KB
Directory structure:
gitextract_pwtx6b4h/
├── .github/
│ ├── FUNDING.yml
│ └── workflows/
│ └── test.yml
├── .gitignore
├── CHANGELOG.md
├── Gemfile
├── LICENSE
├── README.md
├── Rakefile
├── TODO.md
├── benchmarks/
│ └── bm_http1_parser.rb
├── bin/
│ ├── benchmark
│ ├── h1pd
│ └── tipi
├── df/
│ ├── agent.rb
│ ├── etc_benchmark.rb
│ ├── multi_agent_supervisor.rb
│ ├── multi_client.rb
│ ├── routing_benchmark.rb
│ ├── sample_agent.rb
│ ├── server.rb
│ ├── server_utils.rb
│ ├── sse_page.html
│ ├── stress.rb
│ └── ws_page.html
├── docs/
│ └── README.md
├── examples/
│ ├── cuba.ru
│ ├── full_service.rb
│ ├── hanami-api.ru
│ ├── hello.rb
│ ├── hello.ru
│ ├── http1_parser.rb
│ ├── http_request_ws_server.rb
│ ├── http_server.js
│ ├── http_server.rb
│ ├── http_server_forked.rb
│ ├── http_server_form.rb
│ ├── http_server_graceful.rb
│ ├── http_server_routes.rb
│ ├── http_server_simple.rb
│ ├── http_server_static.rb
│ ├── http_server_throttled.rb
│ ├── http_server_throttled_accept.rb
│ ├── http_server_timeout.rb
│ ├── http_unix_socket_server.rb
│ ├── http_ws_server.rb
│ ├── https_server.rb
│ ├── https_server_forked.rb
│ ├── https_wss_server.rb
│ ├── rack_server.rb
│ ├── rack_server_forked.rb
│ ├── rack_server_https.rb
│ ├── rack_server_https_forked.rb
│ ├── routing_server.rb
│ ├── servername_cb.rb
│ ├── source.rb
│ ├── streaming.rb
│ ├── websocket_client.rb
│ ├── websocket_demo.rb
│ ├── websocket_secure_server.rb
│ ├── websocket_server.rb
│ ├── ws_page.html
│ ├── wss_page.html
│ └── zlib-bench.rb
├── lib/
│ ├── tipi/
│ │ ├── acme.rb
│ │ ├── cli.rb
│ │ ├── config_dsl.rb
│ │ ├── configuration.rb
│ │ ├── controller/
│ │ │ ├── bare_polyphony.rb
│ │ │ ├── bare_stock.rb
│ │ │ ├── extensions.rb
│ │ │ ├── stock_http1_adapter.rb
│ │ │ ├── web_polyphony.rb
│ │ │ └── web_stock.rb
│ │ ├── controller.rb
│ │ ├── digital_fabric/
│ │ │ ├── agent.rb
│ │ │ ├── agent_proxy.rb
│ │ │ ├── executive/
│ │ │ │ └── index.html
│ │ │ ├── executive.rb
│ │ │ ├── protocol.rb
│ │ │ ├── request_adapter.rb
│ │ │ └── service.rb
│ │ ├── digital_fabric.rb
│ │ ├── handler.rb
│ │ ├── http1_adapter.rb
│ │ ├── http2_adapter.rb
│ │ ├── http2_stream.rb
│ │ ├── rack_adapter.rb
│ │ ├── response_extensions.rb
│ │ ├── supervisor.rb
│ │ ├── version.rb
│ │ └── websocket.rb
│ └── tipi.rb
├── test/
│ ├── coverage.rb
│ ├── eg.rb
│ ├── helper.rb
│ ├── run.rb
│ ├── test_http_server.rb
│ └── test_request.rb
└── tipi.gemspec
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/FUNDING.yml
================================================
github: noteflakes
================================================
FILE: .github/workflows/test.yml
================================================
name: Tests
on: [push, pull_request]
jobs:
build:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
ruby: ['3.2', '3.3', '3.4', 'head']
name: >-
${{matrix.os}}, ${{matrix.ruby}}
runs-on: ${{matrix.os}}
# env:
# POLYPHONY_LIBEV: "1"
steps:
- name: Setup machine
uses: actions/checkout@v1
- name: Setup Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: ${{matrix.ruby}}
bundler-cache: true # 'bundle install' and cache
cache-version: 2
- name: Run tests
run: bundle exec rake test
================================================
FILE: .gitignore
================================================
*.gem
*.rbc
/.config
/coverage/
/InstalledFiles
/pkg/
/spec/reports/
/spec/examples.txt
/test/tmp/
/test/version_tmp/
/tmp/
# Used by dotenv library to load environment variables.
# .env
# Ignore Byebug command history file.
.byebug_history
## Specific to RubyMotion:
.dat*
.repl_history
build/
*.bridgesupport
build-iPhoneOS/
build-iPhoneSimulator/
## Specific to RubyMotion (use of CocoaPods):
#
# We recommend against adding the Pods directory to your .gitignore. However
# you should judge for yourself, the pros and cons are mentioned at:
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
#
# vendor/Pods/
## Documentation cache and generated files:
/.yardoc/
/_yardoc/
/doc/
/rdoc/
## Environment normalization:
/.bundle/
/vendor/bundle
/lib/bundler/man/
# for a library or gem, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# Gemfile.lock
# .ruby-version
# .ruby-gemset
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
.rvmrc
# Used by RuboCop. Remote config files pulled in from inherit_from directive.
# .rubocop-https?--*
log
log.*
lib/tipi_ext*
examples/certificate_store.db
================================================
FILE: CHANGELOG.md
================================================
## 0.56 2025-10-21
- Update localhost, acme-client, websocket, http-2, extralite, qeweney, rack, bundler dependencies (#35)
- Drop support for Ruby 3.1 (#35)
- Add support for Ruby 3.3 and 3.4 (#35)
## 0.55 2023-07-29
- Simplify HTTP/1 exception handling
- Update H1P dependency
- Update Extralite dependency (#28)
## 0.54 2023-05-28
- Use `H1P.send_response` for sending response
- Update Polyphony and H1P versions
## 0.53 2022-10-04
- Disregard `SystemCallError` in `Tipi.client_loop`
## 0.52 2022-03-03
- Treat HTTP/2 headers as immutable
## 0.51 2022-02-28
- Update dependencies
## 0.50 2022-02-10
- Update Qeweney
## 0.49 2022-02-07
- Update Polyphony
## 0.48 2022-02-04
- Update dependencies
- Fix variable name in `Tipi.verify_path` (#16) - thanks @dm1try
## 0.47 2202-02-03
- Update H1P dependency
## 0.46 2022-02-01
- Allow setting valid hosts
- Change interface of Qeweney apps to use #run (#15)
- Close server listener before terminating connections
## 0.45 2021-10-25
- Remove `http_parser.rb` dependency (#14) - thanks @SwagDevOps
- Use `argv` argument in `Tipi.opts_from_argv` (#13) - thanks @SwagDevOps
- Ignore `ArgumentError` in `#parse_headers`
## 0.44 2021-09-29
- Implement compatibility mode for HTTP/1 (WIP)
- Add option parsing for CLI tool
- Implement supervisor-controller-worker model in CLI tool
## 0.43 2021-08-20
- Extract HTTP/1 parser into a separate gem:
[H1P](https://github.com/digital-fabric/h1p)
## 0.42 2021-08-16
- HTTP/1 parser: disable UTF-8 parsing for all but header values
- Add support for parsing HTTP/1 from callable source
- Introduce full_service API for automatic HTTPS
- Introduce automatic SSL certificate provisioning
- Improve handling of exceptions
- Various fixes to DF service and agent pxoy
- Fix upgrading to HTTP2 with a request body
- Switch to new HTTP/1 parser
## 0.41 2021-07-26
- Fix Rack adapter (#11)
- Introduce experimental HTTP/1 parser
- More work on DF server
- Allow setting chunk size in `#respond_from_io`
## 0.40 2021-06-24
- Implement serving static files using splice_chunks (nice performance boost for
files bigger than 1M)
- Call shutdown before closing socket
- Fix examples (thanks @timhatch!)
## 0.39 2021-06-20
- More work on DF server
- Fix HTTP2StreamHandler#send_headers
- Various fixes to HTTP/2 adapter
- Fix host detection for HTTP/2 connections
- Fix HTTP/1 adapter #respond with nil body
- Fix HTTP1Adapter#send_headers
## 0.38 2021-03-09
- Don't use chunked transfer encoding for non-streaming responses
## 0.37.2 2021-03-08
- Fix header formatting when header value is an array
## 0.37 2021-02-15
- Update upgrade mechanism to work with updated Qeweney API
## 0.36 2021-02-12
- Use `Qeweney::Status` constants
## 0.35 2021-02-10
- Extract Request class into separate [qeweney](https://github.com/digital-fabric/qeweney) gem
## 0.34 2021-02-07
- Implement digital fabric service and agents
- Add multipart and urlencoded form data parsing
- Improve request body reading behaviour
- Add more `Request` information methods
- Add access to connection for HTTP2 requests
- Allow calling `Request#send_chunk` with empty chunk
- Add support for handling protocol upgrades from within request handler
## 0.33 2020-11-20
- Update code for Polyphony 0.47.5
- Add support for Rack::File body to Tipi::RackAdapter
## 0.32 2020-08-14
- Respond with array of strings instead of concatenating for HTTP 1
- Use read_loop instead of readpartial
- Fix http upgrade test
## 0.31 2020-07-28
- Fix websocket server code
- Implement configuration layer (WIP)
- Improve performance of rack adapter
## 0.30 2020-07-15
- Rename project to Tipi
- Rearrange source code
- Remove HTTP client code (to be developed eventually into a separate gem)
- Fix header rendering in rack adapter (#2)
## 0.29 2020-07-06
- Use IO#read_loop
## 0.28 2020-07-03
- Update with API changes from Polyphony >= 0.41
## 0.27 2020-04-14
- Remove modulation dependency
## 0.26 2020-03-03
- Fix `Server#listen`
## 0.25 2020-02-19
- Ensure server socket is closed upon stopping loop
- Fix `Request#format_header_lines`
## 0.24 2020-01-08
- Move HTTP to separate polyphony-http gem
For earlier changes look at the Polyphony changelog.
================================================
FILE: Gemfile
================================================
source 'https://rubygems.org'
gemspec
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2021 Sharon Rosner
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
<p align="center"><img src="docs/tipi-logo.png" /></p>
# Tipi - the All-in-one Web Server for Ruby Apps
[](http://rubygems.org/gems/tipi)
[](https://github.com/digital-fabric/tipi/actions?query=workflow%3ATests)
[](https://github.com/digital-fabric/tipi/blob/master/LICENSE)
## What is Tipi?
Tipi is an integrated, feature-complete HTTP/S server for Ruby applications.
Tipi is built on top of
[Polyphony](https://github.com/digital-fabric/polyphony), a robust,
high-performance library for building highly-concurrent applications in Ruby.
Tipi can be used to serve any Rack application or set of static files directly
without having to employ a reverse-proxy such as Nginx.
## Features
* High-performance, highly concurrent web server based on
[Polyphony](https://github.com/digital-fabric/polyphony)
* Full support for HTTP/1, HTTP/2, WebSocket protocols
* Built-in SSL termination for secure, encrypted connections
* **Automatic SSL certificates** using ACME providers such as Let's Encrypt (WIP)
* Automatic ALPN protocol selection for serving HTTP/2
* Request and response body streaming for efficient downloads and uploads
* Full support for Rack-based apps
## Benchmarks
> Caveat emptor: the following results were obtained with an ad-hoc, manual
> process. I am not really familiar with the servers I compared Tipi against,
> and I ran them in their default configuration (apart from setting the number
> of workers). Take these results with a bunch of salt.
<img src="bm.png" style="width: 480px">
| |Tipi|Puma|Falcon|Unicorn|
|-|---:|---:|-----:|------:|
|HTTP/1.1|138629|34573|40714|7438|
|HTTPS/2|56762|n/a|34226|n/a|
### Methodology
- All servers ran the same "Hello world" [Rack
application](https://github.com/digital-fabric/tipi/blob/master/examples/hello.ru)
- Each server was run with 4 forked worker processes:
- Tipi: `tipi -w4 -flocalhost:10080:10443 examples/hello.ru`
- [Puma](https://github.com/puma/puma): `puma -w 4 examples/hello.ru`
- [Falcon](https://github.com/socketry/falcon/): `falcon -n 4 -b http://localhost:9292/ -c examples/hello.ru`
- [Unicorn](https://yhbt.net/unicorn/): `unicorn -c u.conf examples/hello.ru`
with the configuration file containing the directive `worker_processes 4`
- The benchmark results were obtained using `wrk -d60 -t4 -c64 <url>`
- All servers were run on Ruby 2.7.2p137
- Machine specs: i5-8350U@1.7GHzx8 CPU, 8GB of RAM, running Linux kernel version 5.13.7
- Puma does not support HTTP/2.
- As far as I could tell Unicorn does not support SSL termination.
## Running Tipi
To run Tipi, run the included `tipi` command. Alternatively you can add tipi as
a dependency to your Gemfile, then run `bundle exec tipi`. By default
Tipi can be used to drive Rack apps or alternatively any app using the
[Qeweney](https://github.com/digital-fabric/qeweney) request-response interface.
### Running Rack apps
Use the `tipi` command to start your app:
```bash
$ bundle exec tipi myapp.ru
```
### Running Qeweney apps
```bash
$ bundle exec tipi myapp.rb
```
The app script file should define an `app` method that returns a proc/lambda
taking a single `Qeweney::Request` argument. Here's an example:
```ruby
# frozen_string_literal: true
def app
->(req) { req.respond('Hello, world!', 'Content-Type' => 'text/plain') }
end
```
## Setting server listening options
By default, Tipi serves plain HTTP on port 1234, but you can easily change that
by providing command line options as follows:
### HTTP
To listen for plain HTTP, use the `-l`/`--listen` option and specify a port
number:
```bash
$ bundle exec tipi -l9292 myapp.ru
```
### HTTPS
To listen for HTTPS connections, use the `-s`/`--secure` option and specify a
host name and a port:
```bash
$ bundle exec tipi -sexample.com:9292 myapp.ru
```
### Full service listening
The Tipi full service listens for both HTTP and HTTPS and supports automatic
certificate provisioning. To use the full service, use the `-f`/`--full` option,
and specify the domain name, the HTTP port, and the HTTPS port, e.g.:
```bash
$ bundle exec tipi -fmysite.org:10080:10443 myapp.ru
#If serving multiple domains, you can use * as place holder
$ bundle exec tipi -f*:10080:10443 myapp.ru
```
If `localhost` is specified as the domain, Tipi will automatically generate a
localhost certificate.
## Concurrency settings
By default, the `tipi` command starts a single controller and uses
[Polyphony](https://github.com/digital-fabric/polyphony) to run each connection
on its own fiber. This means that you will have a single process running on a
single thread (on a single CPU core). In order to parallelize your app and
employ multiple CPU cores, you can tell Tipi to fork multiple worker processes
to run your app. The number of workers is controlled using the `-w`/`--workers`
option:
```bash
# fork 4 worker processes
$ bundle exec tipi -w4 myapp.ru
```
You can also set Tipi to spawn multiple threads in each worker when in
compatibility mode (see below.)
## Compatibility mode
> Note: compatibility mode is still being developed, and currently only supports
> HTTP/1 connections.
In some apps, using Polyphony is not possible, due to incompatibilities between
it and other third-party dependencies. In order to be able to run these apps,
Tipi provides a compatibility mode that does not use Polyphony for concurrency,
but instead uses a thread-per-connection concurrency model. You can also fork
multiple workers, each running multiple threads, if so desired. Note that the
concurrency level is the maximum number workers multiplied by the number of
threads per worker:
```
concurrency = worker_count * threads_per_worker
```
To run Tipi in compatibility mode, use the `-c`/`--compatibility` option, e.g.:
```bash
# 4 workers * 8 threads = 32 max concurrency
$ bundle exec tipi -c -w4 -t8 myapp.ru
```
## Worker process supervision
Tipi employs a supervisor-controller-worker process supervision model, which
minimizes the memory consumption of forked workers, and which facilitates
graceful reloading after updating the application code.
This supervision model is made of three levels:
- Supervisor - Starts and stops the controller process
- Controller - loads the application code and forks workers
- Worker - listens for connections, handles incoming requests
(If the worker count is 1, the Controller and Worker roles are merged into a
single process.)
This model allows Tipi to fork workers after loading the app code, and use a
much simpler way to perform graceful restarts:
- The supervisor starts a new controller process (which may fork one or more
worker processes).
- Sleep for a certain amount of time (currently 1 second.)
- Stop the old controller process.
- Each worker process is gracefully stopped and allowed to finish all pending
requests, then shutdown all open connections.
## Performing a graceful restart
A graceful restart performed by sending `SIGUSR2` to the supervisor process.
## Documentation
Documentation for Tipi's API is coming soon...
================================================
FILE: Rakefile
================================================
# frozen_string_literal: true
require "bundler/gem_tasks"
require "rake/clean"
task :default => [:test]
task :test do
exec 'ruby test/run.rb'
end
================================================
FILE: TODO.md
================================================
## Rethink design
- Remove DF code
- Remove non-Polyphony code
# Miscellaneous
- Try using `TCP_DEFER_ACCEPT` with Polyphony on io_uring - does it provide any
performance benefit?
# What about HTTP/2?
It would be a nice exercise in converting a callback-based API to a blocking
one:
```ruby
parser = Tipi::HTTP2::Parser.new(socket)
parser.each_stream(socket) do |stream|
spin { handle_stream(stream) }
end
```
# Roadmap
- Improve Rack spec compliance, add tests
- Homogenize HTTP 1 and HTTP 2 headers - downcase symbols
- Use `http-2-next` instead of `http-2` for http/2
- https://gitlab.com/honeyryderchuck/http-2-next
- Open an issue there, ask what's the difference between the two gems?
## 0.38
- Add more poly CLI commands and options:
- serve static files from given directory
- serve from rack up file
- serve both http and https
- use custom certificate files for SSL
- set host address to bind to
- set port to bind to
- set forking process count
## 0.39 Working Sinatra application
- app with database access (postgresql)
- benchmarks!
================================================
FILE: benchmarks/bm_http1_parser.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"
def measure_time_and_allocs
4.times { GC.start }
GC.disable
t0 = Time.now
a0 = object_count
yield
t1 = Time.now
a1 = object_count
[t1 - t0, a1 - a0]
ensure
GC.enable
end
def object_count
count = ObjectSpace.count_objects
count[:TOTAL] - count[:FREE]
end
def benchmark_other_http1_parser(iterations)
STDOUT << "http_parser.rb: "
require 'http_parser.rb'
i, o = IO.pipe
parser = Http::Parser.new
done = false
headers = nil
parser.on_headers_complete = proc do |h|
headers = h
headers[':method'] = parser.http_method
headers[':path'] = parser.request_url
end
parser.on_message_complete = proc { done = true }
elapsed, allocated = measure_time_and_allocs do
iterations.times do
o << HTTP_REQUEST
done = false
while !done
msg = i.readpartial(4096)
parser << msg
end
end
end
puts(format('elapsed: %f, allocated: %d (%f/req), rate: %f ips', elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
end
def benchmark_tipi_http1_parser(iterations)
STDOUT << "tipi parser: "
require_relative '../lib/tipi_ext'
i, o = IO.pipe
reader = proc { |len| i.readpartial(len) }
parser = Tipi::HTTP1Parser.new(reader)
elapsed, allocated = measure_time_and_allocs do
iterations.times do
o << HTTP_REQUEST
headers = parser.parse_headers
end
end
puts(format('elapsed: %f, allocated: %d (%f/req), rate: %f ips', elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
end
def fork_benchmark(method, iterations)
pid = fork do
send(method, iterations)
rescue Exception => e
p e
p e.backtrace
exit!
end
Process.wait(pid)
end
x = 500000
# fork_benchmark(:benchmark_other_http1_parser, x)
# fork_benchmark(:benchmark_tipi_http1_parser, x)
benchmark_tipi_http1_parser(x)
================================================
FILE: bin/benchmark
================================================
#!/usr/bin/env ruby
require 'bundler/setup'
require 'polyphony'
def parse_latency(latency)
m = latency.match(/^([\d\.]+)(us|ms|s)$/)
return nil unless m
value = m[1].to_f
case m[2]
when 's' then value
when 'ms' then value / 1000
when 'us' then value / 1000000
end
end
def parse_wrk_results(results)
lines = results.lines
latencies = lines[3].strip.split(/\s+/)
throughput = lines[6].strip.split(/\s+/)
{
latency_avg: parse_latency(latencies[1]),
latency_max: parse_latency(latencies[3]),
rate: throughput[1].to_f
}
end
def run_wrk(duration: 10, threads: 2, connections: 10, url: )
`wrk -d#{duration} -t#{threads} -c#{connections} #{url}`
end
[8, 64, 256, 512].each do |c|
puts "connections: #{c}"
p parse_wrk_results(run_wrk(duration: 10, threads: 4, connections: c, url: "http://localhost:10080/"))
end
================================================
FILE: bin/h1pd
================================================
#!/usr/bin/env bash
set -e
rake compile
ruby test/test_http1_parser.rb
ruby benchmarks/bm_http1_parser.rb
================================================
FILE: bin/tipi
================================================
#!/usr/bin/env ruby
require 'bundler/setup'
require 'tipi/cli'
trap('SIGINT') { exit }
Tipi::CLI.start
================================================
FILE: df/agent.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'polyphony'
require 'json'
require 'tipi/digital_fabric/protocol'
require 'tipi/digital_fabric/agent'
Protocol = DigitalFabric::Protocol
class SampleAgent < DigitalFabric::Agent
def initialize(id, server_url)
@id = id
super(server_url, { host: "#{id}.realiteq.net" }, 'foobar')
@name = "agent-#{@id}"
end
def http_request(req)
return streaming_http_request(req) if req.path == '/streaming'
return form_http_request(req) if req.path == '/form'
req.respond({ id: @id, time: Time.now.to_i }.to_json)
end
def streaming_http_request(req)
req.send_headers({ 'Content-Type': 'text/json' })
60.times do
sleep 1
do_some_activity
req.send_chunk({ id: @id, time: Time.now.to_i }.to_json)
end
req.finish
rescue Polyphony::Terminate
req.respond(' * shutting down *') if Fiber.current.graceful_shutdown?
rescue Exception => e
p e
puts e.backtrace.join("\n")
end
def form_http_request(req)
body = req.read
form_data = Tipi::Request.parse_form_data(body, req.headers)
req.respond({ form_data: form_data, headers: req.headers }.to_json, { 'Content-Type': 'text/json' })
end
def do_some_activity
File.open('/tmp/df-test.log', 'a+') { |f| sleep rand; f.puts "#{Time.now} #{@name} #{generate_data(2**8)}" }
end
def generate_data(length)
charset = Array('A'..'Z') + Array('a'..'z') + Array('0'..'9')
Array.new(length) { charset.sample }.join
end
end
# id = ARGV[0]
# puts "Starting agent #{id} pid: #{Process.pid}"
# spin_loop(interval: 60) { GC.start }
# SampleAgent.new(id, '/tmp/df.sock').run
# SampleAgent.new(id, 'localhost:4411').run
================================================
FILE: df/etc_benchmark.rb
================================================
# frozen_string_literal: true
require 'securerandom'
def generate
SecureRandom.uuid
end
count = 100000
GC.disable
t0 = Time.now
count.times { generate }
elapsed = Time.now - t0
puts "rate: #{count / elapsed}/s"
================================================
FILE: df/multi_agent_supervisor.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'polyphony'
require 'json'
require 'fileutils'
FileUtils.cd(__dir__)
require_relative 'agent'
class AgentManager
def initialize
@running_agents = {}
@pending_actions = Queue.new
@processor = spin_loop { process_pending_action }
end
def process_pending_action
action = @pending_actions.shift
case action[:kind]
when :start
start_agent(action[:spec])
when :stop
stop_agent(action[:spec])
end
end
def start_agent(spec)
return if @running_agents[spec]
@running_agents[spec] = spin do
while true
launch_agent_from_spec(spec)
sleep 1
end
ensure
@running_agents.delete(spec)
end
end
def stop_agent(spec)
fiber = @running_agents[spec]
return unless fiber
fiber.terminate
fiber.await
end
def update
return unless @pending_actions.empty?
current_specs = @running_agents.keys
updated_specs = agent_specs
to_start = updated_specs - current_specs
to_stop = current_specs - current_specs
to_start.each { |s| @pending_actions << { kind: :start, spec: s } }
to_stop.each { |s| @pending_actions << { kind: :stop, spec: s } }
end
def run
every(2) { update }
end
end
class RealityAgentManager < AgentManager
def agent_specs
(1..400).map { |i| { id: i } }
end
def launch_agent_from_spec(spec)
# Polyphony::Process.watch("ruby agent.rb #{spec[:id]}")
Polyphony::Process.watch do
spin_loop(interval: 60) { GC.start }
agent = SampleAgent.new(spec[:id], '/tmp/df.sock')
puts "Starting agent #{spec[:id]} pid: #{Process.pid}"
agent.run
end
end
end
puts "Agent manager pid: #{Process.pid}"
manager = RealityAgentManager.new
manager.run
================================================
FILE: df/multi_client.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'polyphony'
require 'http/parser'
class Client
def initialize(id, host, port, http_host, interval)
@id = id
@host = host
@port = port
@http_host = http_host
@interval = interval.to_f
@interval_delta = @interval / 2
end
def run
while true
connect && issue_requests
sleep 5
end
end
def connect
@socket = Polyphony::Net.tcp_connect(@host, @port)
rescue SystemCallError
false
end
REQUEST = <<~HTTP
GET / HTTP/1.1
Host: %s
HTTP
def issue_requests
@parser = Http::Parser.new
@parser.on_message_complete = proc { @got_reply = true }
@parser.on_body = proc { |chunk| @response = chunk }
while true
do_request
sleep rand((@interval - @interval_delta)..(@interval + @interval_delta))
end
rescue IOError, Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNREFUSED => e
# fail quitely
ensure
@parser = nil
end
def do_request
@got_reply = nil
@response = nil
@socket << format(REQUEST, @http_host)
wait_for_response
# if @parser.status_code != 200
# puts "Got status code #{@parser.status_code} from #{@http_host} => #{@parser.headers && @parser.headers['X-Request-ID']}"
# end
# puts "#{Time.now} [client-#{@id}] #{@http_host} => #{@response || '<error>'}"
end
def wait_for_response
@socket.recv_loop do |data|
@parser << data
return @response if @got_reply
end
end
end
def spin_client(id, host)
spin do
client = Client.new(id, 'localhost', 4411, host, 30)
client.run
end
end
spin_loop(interval: 60) { GC.start }
10000.times { |id| spin_client(id, "#{rand(1..400)}.realiteq.net") }
trap('SIGINT') { exit! }
puts "Multi client pid: #{Process.pid}"
sleep
================================================
FILE: df/routing_benchmark.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'polyphony'
require 'tipi/digital_fabric'
class FakeAgent
def initialize(idx)
@idx = idx
end
end
def setup_df_service_with_agents(agent_count)
server = DigitalFabric::Service.new
agent_count.times do |i|
server.mount({path: "/#{i}"}, FakeAgent.new(i))
end
server
end
def benchmark_route_compilation(agent_count, iterations)
service = setup_df_service_with_agents(agent_count)
t0 = Time.now
iterations.times { service.compile_agent_routes }
elapsed = Time.now - t0
puts "route_compilation: #{agent_count} => #{elapsed / iterations}s (#{1/(elapsed / iterations)} ops/sec)"
end
class FauxRequest
def initialize(agent_count)
@agent_count = agent_count
end
def headers
{ ':path' => "/#{rand(@agent_count)}"}
end
end
def benchmark_find_agent(agent_count, iterations)
service = setup_df_service_with_agents(agent_count)
t0 = Time.now
request = FauxRequest.new(agent_count)
iterations.times do
agent = service.find_agent(request)
end
elapsed = Time.now - t0
puts "routing: #{agent_count} => #{elapsed / iterations}s (#{1/(elapsed / iterations)} ops/sec)"
end
def benchmark
benchmark_route_compilation(100, 1000)
benchmark_route_compilation(500, 200)
benchmark_route_compilation(1000, 100)
benchmark_find_agent(100, 1000)
benchmark_find_agent(500, 200)
benchmark_find_agent(1000, 100)
end
benchmark
================================================
FILE: df/sample_agent.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'polyphony'
require 'json'
require 'tipi/digital_fabric/protocol'
require 'tipi/digital_fabric/agent'
Protocol = DigitalFabric::Protocol
class SampleAgent < DigitalFabric::Agent
HTML_WS = IO.read(File.join(__dir__, 'ws_page.html'))
HTML_SSE = IO.read(File.join(__dir__, 'sse_page.html'))
def http_request(req)
path = req.headers[':path']
case path
when '/agent'
send_df_message(Protocol.http_response(
req['id'],
'Hello, world!',
{},
true
))
when '/agent/ws'
send_df_message(Protocol.http_response(
req['id'],
HTML_WS,
{ 'Content-Type' => 'text/html' },
true
))
when '/agent/sse'
send_df_message(Protocol.http_response(
req['id'],
HTML_SSE,
{ 'Content-Type' => 'text/html' },
true
))
when '/agent/sse/events'
stream_sse_response(req)
else
send_df_message(Protocol.http_response(
req['id'],
nil,
{ ':status' => 400 },
true
))
end
end
def ws_request(req)
send_df_message(Protocol.ws_response(req['id'], {}))
10.times do
sleep 1
send_df_message(Protocol.ws_data(req['id'], Time.now.to_s))
end
send_df_message(Protocol.ws_close(req['id']))
end
def stream_sse_response(req)
send_df_message(Protocol.http_response(
req['id'],
nil,
{ 'Content-Type' => 'text/event-stream' },
false
))
10.times do
sleep 1
send_df_message(Protocol.http_response(
req['id'],
"data: #{Time.now}\n\n",
nil,
false
))
end
send_df_message(Protocol.http_response(
req['id'],
"retry: 0\n\n",
nil,
true
))
end
end
agent = SampleAgent.new('127.0.0.1', 4411, { path: '/agent' })
agent.run
================================================
FILE: df/server.rb
================================================
# frozen_string_literal: true
require_relative 'server_utils'
listeners = [
listen_http,
listen_https,
listen_unix
]
spin_loop(interval: 60) { GC.compact } if GC.respond_to?(:compact)
begin
log('Starting DF server')
Fiber.await(*listeners)
rescue Interrupt
log('Got SIGINT, shutting down gracefully')
@service.graceful_shutdown
rescue SystemExit
# ignore
rescue Exception => e
log("Uncaught exception", error: e, source: e.source_fiber, raising: e.raising_fiber, backtrace: e.backtrace)
ensure
log('DF server stopped')
end
================================================
FILE: df/server_utils.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'tipi/digital_fabric'
require 'tipi/digital_fabric/executive'
require 'json'
require 'fileutils'
require 'time'
require 'polyphony/extensions/debug'
FileUtils.cd(__dir__)
@service = DigitalFabric::Service.new(token: 'foobar')
@executive = DigitalFabric::Executive.new(@service, { host: '@executive.realiteq.net' })
@pid = Process.pid
def log(msg, **ctx)
text = format(
"%s (%d) %s\n",
Time.now.strftime('%Y-%m-%d %H:%M:%S.%3N'),
@pid,
msg
)
STDOUT.orig_write text
return if ctx.empty?
ctx.each { |k, v| STDOUT.orig_write format(" %s: %s\n", k, v.inspect) }
end
def listen_http
spin(:http_listener) do
opts = {
reuse_addr: true,
dont_linger: true,
}
log('Listening for HTTP on localhost:10080')
server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
id = 0
loop do
client = server.accept
# log("Accept HTTP connection", client: client)
spin("http#{id += 1}") do
@service.incr_connection_count
Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
ensure
# log("Done with HTTP connection", client: client)
@service.decr_connection_count
end
rescue Polyphony::BaseException
raise
rescue Exception => e
log 'HTTP accept (unknown) error', error: e, backtrace: e.backtrace
end
end
end
CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
def listen_https
spin(:https_listener) do
private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
c = IO.read('../../reality/ssl/cacert.pem')
certificates = c.scan(CERTIFICATE_REGEXP).map { |p| OpenSSL::X509::Certificate.new(p.first) }
ctx = OpenSSL::SSL::SSLContext.new
ctx.security_level = 0
cert = certificates.shift
log "SSL Certificate expires: #{cert.not_after.inspect}"
ctx.add_certificate(cert, private_key, certificates)
# ctx.ciphers = 'ECDH+aRSA'
ctx.max_version = OpenSSL::SSL::TLS1_3_VERSION
ctx.min_version = OpenSSL::SSL::SSL3_VERSION
ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
# TODO: further limit ciphers
# ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
# ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/tls.rb
opts = {
reuse_addr: true,
dont_linger: true,
secure_context: ctx,
alpn_protocols: Tipi::ALPN_PROTOCOLS
}
log('Listening for HTTPS on localhost:10443')
server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
id = 0
loop do
client = server.accept rescue nil
next unless client
# log('Accept HTTPS client connection', client: client)
spin("https#{id += 1}") do
@service.incr_connection_count
Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
rescue => e
log('Error while handling HTTPS client', client: client, error: e, backtrace: e.backtrace)
ensure
# log("Done with HTTP connection", client: client)
@service.decr_connection_count
end
# rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError => e
# log('HTTPS accept error', error: e)
rescue Polyphony::BaseException
raise
rescue Exception => e
log 'HTTPS listener error: ', error: e, backtrace: e.backtrace
end
end
end
UNIX_SOCKET_PATH = '/tmp/df.sock'
def listen_unix
spin(:unix_listener) do
log("Listening on #{UNIX_SOCKET_PATH}")
FileUtils.rm(UNIX_SOCKET_PATH) if File.exists?(UNIX_SOCKET_PATH)
socket = UNIXServer.new(UNIX_SOCKET_PATH)
id = 0
loop do
client = socket.accept
# log('Accept Unix connection', client: client)
spin("unix#{id += 1}") do
Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
end
rescue Polyphony::BaseException
raise
rescue Exception => e
log 'Unix accept error', error: e, backtrace: e.backtrace
end
end
end
def listen_df
spin(:df_listener) do
opts = {
reuse_addr: true,
reuse_port: true,
dont_linger: true,
}
log('Listening for DF connections on localhost:4321')
server = Polyphony::Net.tcp_listen('0.0.0.0', 4321, opts)
id = 0
loop do
client = server.accept
# log('Accept DF connection', client: client)
spin("df#{id += 1}") do
Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
end
rescue Polyphony::BaseException
raise
rescue Exception => e
log 'DF accept (unknown) error', error: e, backtrace: e.backtrace
end
end
end
if ENV['TRACE'] == '1'
Thread.backend.trace_proc = proc do |event, fiber, value, pri|
fiber_id = fiber.tag || fiber.inspect
case event
when :schedule
log format("=> %s %s %s %s", event, fiber_id, value.inspect, pri ? '(priority)' : '')
when :unblock
log format("=> %s %s %s", event, fiber_id, value.inspect)
when :spin, :terminate
log format("=> %s %s", event, fiber_id)
else
log format("=> %s", event)
end
end
end
================================================
FILE: df/sse_page.html
================================================
<!doctype html>
<html lang="en">
<head>
<title>SSE Client</title>
</head>
<body>
<h1>SSE Client</h1>
<script>
var connect = function () {
console.log("connecting...");
var eventSource = new EventSource("/agent/sse/events");
eventSource.addEventListener('open', function(e) {
console.log("connected");
document.querySelector('#status').innerText = 'connected';
return false;
}, false);
eventSource.addEventListener('message', function(e) {
document.querySelector('#msg').innerText = e.data;
}, false);
};
window.onload = connect;
</script>
<h1 id="status">disconnected</h1>
<h1 id="msg"></h1>
</body>
</html>
================================================
FILE: df/stress.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'polyphony'
require 'fileutils'
FileUtils.cd(__dir__)
def monitor_process(cmd)
while true
puts "Starting #{cmd}"
Polyphony::Process.watch(cmd)
sleep 5
end
end
puts "pid: #{Process.pid}"
puts 'Starting stress test'
spin { monitor_process('ruby server.rb') }
spin { monitor_process('ruby multi_agent_supervisor.rb') }
spin { monitor_process('ruby multi_client.rb') }
sleep
================================================
FILE: df/ws_page.html
================================================
<!doctype html>
<html lang="en">
<head>
<title>Websocket Client</title>
</head>
<body>
<h1>WebSocket Client</h1>
<script>
var connect = function () {
console.log("connecting...")
var exampleSocket = new WebSocket("wss://dev.realiteq.net/agent");
exampleSocket.onopen = function (event) {
console.log("connected");
document.querySelector('#status').innerText = 'connected';
exampleSocket.send("Can you hear me?");
};
exampleSocket.onclose = function (event) {
console.log("disconnected");
document.querySelector('#status').innerText = 'disconnected';
setTimeout(function () {
// exampleSocket.removeAllListeners();
connect();
}, 1000);
}
exampleSocket.onmessage = function (event) {
console.log("got message", event.data);
document.querySelector('#msg').innerText = event.data;
console.log(event.data);
}
};
window.onload = connect;
</script>
<h1 id="status">disconnected</h1>
<h1 id="msg"></h1>
</body>
</html>
================================================
FILE: docs/README.md
================================================
# Polyphony - Easy Concurrency for Ruby
> Polyphony \| pəˈlɪf\(ə\)ni \|
> 1. _Music_ the style of simultaneously combining a number of parts, each
> forming an individual melody and harmonizing with each other.
> 2. _Programming_ a Ruby gem for concurrent programming focusing on performance
> and developer happiness.
Polyphony is a library for building concurrent applications in Ruby. Polyphony
harnesses the power of [Ruby fibers](https://ruby-doc.org/core-2.5.1/Fiber.html)
to provide a cooperative, sequential coprocess-based concurrency model. Under
the hood, Polyphony uses [libev](https://github.com/enki/libev) as a
high-performance event reactor that provides timers, I/O watchers and other
asynchronous event primitives.
Polyphony makes it possible to use normal Ruby built-in classes like `IO`, and
`Socket` in a concurrent fashion without having to resort to threads. Polyphony
takes care of context-switching automatically whenever a blocking call like
`Socket#accept` or `IO#read` is issued.
## Features
* **Full-blown, integrated, high-performance HTTP 1 / HTTP 2 / WebSocket server
with TLS/SSL termination, automatic ALPN protocol selection, and body
streaming**.
* Co-operative scheduling of concurrent tasks using Ruby fibers.
* High-performance event reactor for handling I/O events and timers.
* Natural, sequential programming style that makes it easy to reason about concurrent code.
* Abstractions and constructs for controlling the execution of concurrent code:
coprocesses, supervisors, throttling, resource pools etc.
* Code can use native networking classes and libraries, growing support for
third-party gems such as `pg` and `redis`.
* Use stdlib classes such as `TCPServer` and `TCPSocket` and `Net::HTTP`.
* HTTP 1 / HTTP 2 client agent with persistent connections.
* Competitive performance and scalability characteristics, in terms of both
throughput and memory consumption.
## Prior Art
Polyphony draws inspiration from the following, in no particular order:
* [nio4r](https://github.com/socketry/nio4r/) and
[async](https://github.com/socketry/async) (Polyphony's C-extension code is
largely a spinoff of
[nio4r's](https://github.com/socketry/nio4r/tree/master/ext))
* [EventMachine](https://github.com/eventmachine/eventmachine)
* [Trio](https://trio.readthedocs.io/)
* [Erlang supervisors](http://erlang.org/doc/man/supervisor.html) (and actually,
Erlang in general)
## Going further
To learn more about using Polyphony to build concurrent applications, read the
technical overview below, or look at the [included
examples](https://github.com/digital-fabric/polyphony/tree/9e0f3b09213156bdf376ef33684ef267517f06e8/examples/README.md).
A thorough reference is forthcoming.
## Contributing to Polyphony
Issues and pull requests will be gladly accepted. Please use the git repository
at https://github.com/digital-fabric/polyphony as your primary point of
departure for contributing.
================================================
FILE: examples/cuba.ru
================================================
# frozen_string_literal: true
require 'cuba'
require 'cuba/safe'
require 'delegate' # See https://github.com/rack/rack/pull/1610
Cuba.use Rack::Session::Cookie, secret: '__a_very_long_string__'
Cuba.plugin Cuba::Safe
Cuba.define do
on get do
on 'hello' do
res.write 'Hello world!'
end
on root do
res.redirect '/hello'
end
end
end
run Cuba
================================================
FILE: examples/full_service.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
::Exception.__disable_sanitized_backtrace__ = true
certificate_db_path = File.expand_path('certificate_store.db', __dir__)
certificate_store = Tipi::ACME::SQLiteCertificateStore.new(certificate_db_path)
Tipi.full_service(
certificate_store: certificate_store
) { |req| req.respond('Hello, world!') }
================================================
FILE: examples/hanami-api.ru
================================================
# frozen_string_literal: true
require 'hanami/api'
class ExampleApi < Hanami::API
get '/hello' do
'Hello world!'
end
get '/' do
redirect '/hello'
end
get '/404' do
404
end
get '/500' do
500
end
end
run ExampleApi.new
================================================
FILE: examples/hello.rb
================================================
# frozen_string_literal: true
run { |req|
req.respond('Hello, world!')
}
================================================
FILE: examples/hello.ru
================================================
# frozen_string_literal: true
run lambda { |env|
[
200,
{"Content-Type" => "text/plain"},
["Hello, world!"]
]
}
================================================
FILE: examples/http1_parser.rb
================================================
# frozen_string_literal: true
require 'polyphony'
require_relative '../lib/tipi_ext'
i, o = IO.pipe
module ::Kernel
def trace(*args)
STDOUT.orig_write(format_trace(args))
end
def format_trace(args)
if args.first.is_a?(String)
if args.size > 1
format("%s: %p\n", args.shift, args)
else
format("%s\n", args.first)
end
else
format("%p\n", args.size == 1 ? args.first : args)
end
end
end
f = spin do
parser = Tipi::HTTP1Parser.new(i)
while true
trace '*' * 40
headers = parser.parse_headers
break unless headers
trace headers
body = parser.read_body
trace "body: #{body ? body.bytesize : 0} bytes"
trace body if body && body.bytesize < 80
end
end
o << "GET /a HTTP/1.1\r\n\r\n"
# o << "GET /a HTTP/1.1\r\nContent-Length: 0\r\n\r\n"
# o << "GET / HTTP/1.1\r\nHost: localhost:10080\r\nUser-Agent: curl/7.74.0\r\nAccept: */*\r\n\r\n"
o << "post /?q=time&blah=blah HTTP/1\r\nTransfer-Encoding: chunked\r\n\r\na\r\nabcdefghij\r\n0\r\n\r\n"
data = " " * 4000000
o << "get /?q=time HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
o << "get /?q=time HTTP/1.1\r\nCookie: foo\r\nCookie: bar\r\n\r\n"
o.close
f.await
================================================
FILE: examples/http_request_ws_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'tipi/websocket'
def ws_handler(conn)
timer = spin_loop(interval: 1) do
conn << Time.now.to_s
end
while (msg = conn.recv)
conn << "you said: #{msg}"
end
ensure
timer.stop
end
opts = {
reuse_addr: true,
dont_linger: true,
}
HTML = IO.read(File.join(__dir__, 'ws_page.html'))
puts "pid: #{Process.pid}"
puts 'Listening on port 4411...'
Tipi.serve('0.0.0.0', 4411, opts) do |req|
if req.upgrade_protocol == 'websocket'
conn = req.upgrade_to_websocket
ws_handler(conn)
else
req.respond(HTML, 'Content-Type' => 'text/html')
end
end
================================================
FILE: examples/http_server.js
================================================
// For the sake of comparing performance, here's a node.js-based HTTP server
// doing roughly the same thing as http_server. Preliminary benchmarking shows
// the ruby version has a throughput (req/s) of about 2/3 of the JS version.
const http = require('http');
const MSG = 'Hello World';
const server = http.createServer((req, res) => {
// let requestCopy = {
// method: req.method,
// request_url: req.url,
// headers: req.headers
// };
// res.writeHead(200, { 'Content-Type': 'application/json' });
// res.end(JSON.stringify(requestCopy));
res.writeHead(200);
res.end(MSG)
});
server.listen(1235);
console.log('Listening on port 1235');
================================================
FILE: examples/http_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
opts = {
reuse_addr: true,
dont_linger: true
}
puts "pid: #{Process.pid}"
puts 'Listening on port 10080...'
# GC.disable
# Thread.current.backend.idle_gc_period = 60
spin_loop(interval: 10) { p Thread.backend.stats }
spin_loop(interval: 10) do
GC.compact
end
spin do
Tipi.serve('0.0.0.0', 10080, opts) do |req|
if req.path == '/stream'
req.send_headers('Foo' => 'Bar')
sleep 1
req.send_chunk("foo\n")
sleep 1
req.send_chunk("bar\n")
req.finish
elsif req.path == '/upload'
body = req.read
req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
else
req.respond("Hello world!\n")
end
# p req.transfer_counts
end
p 'done...'
end.await
================================================
FILE: examples/http_server_forked.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
::Exception.__disable_sanitized_backtrace__ = true
opts = {
reuse_addr: true,
reuse_port: true,
dont_linger: true
}
server = Tipi.listen('0.0.0.0', 1234, opts)
child_pids = []
8.times do
pid = Polyphony.fork do
puts "forked pid: #{Process.pid}"
server.each do |req|
req.respond("Hello world! from pid: #{Process.pid}\n")
end
rescue Interrupt
end
child_pids << pid
end
puts 'Listening on port 1234'
trap('SIGINT') { exit! }
child_pids.each { |pid| Thread.current.backend.waitpid(pid) }
================================================
FILE: examples/http_server_form.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
opts = {
reuse_addr: true,
dont_linger: true
}
puts "pid: #{Process.pid}"
puts 'Listening on port 4411...'
spin do
Tipi.serve('0.0.0.0', 4411, opts) do |req|
body = req.read
body2 = req.read
req.respond("body: #{body} (body2: #{body2.inspect})\n")
rescue Exception => e
p e
end
p 'done...'
end.await
================================================
FILE: examples/http_server_graceful.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'polyphony'
require 'tipi'
opts = {
reuse_addr: true,
dont_linger: true
}
server = spin do
Tipi.serve('0.0.0.0', 1234, opts) do |req|
req.respond("Hello world!\n")
end
end
trap('SIGHUP') do
puts 'got hup'
server.interrupt
end
puts "pid: #{Process.pid}"
puts 'Send HUP to stop gracefully'
puts 'Listening on port 1234...'
suspend
================================================
FILE: examples/http_server_routes.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
opts = {
reuse_addr: true,
dont_linger: true
}
puts "pid: #{Process.pid}"
puts 'Listening on port 4411...'
app = Tipi.route do |req|
req.on 'stream' do
req.send_headers('Foo' => 'Bar')
sleep 1
req.send_chunk("foo\n")
sleep 1
req.send_chunk("bar\n")
req.finish
end
req.default do
req.respond("Hello world!\n")
end
end
trap('INT') { exit! }
Tipi.serve('0.0.0.0', 4411, opts, &app)
================================================
FILE: examples/http_server_simple.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
puts "pid: #{Process.pid}"
puts 'Listening on port 1234...'
Tipi.serve('0.0.0.0', 1234) do |req|
req.respond("Hello world!\n")
end
================================================
FILE: examples/http_server_static.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'fileutils'
opts = {
reuse_addr: true,
dont_linger: true
}
puts "pid: #{Process.pid}"
puts 'Listening on port 4411...'
root_path = FileUtils.pwd
trap('INT') { exit! }
Tipi.serve('0.0.0.0', 4411, opts) do |req|
path = File.join(root_path, req.path)
if File.file?(path)
req.serve_file(path)
else
req.respond(nil, ':status' => Qeweney::Status::NOT_FOUND)
end
end
================================================
FILE: examples/http_server_throttled.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
$throttler = Polyphony::Throttler.new(1000)
opts = { reuse_addr: true, dont_linger: true }
server = spin do
Tipi.serve('0.0.0.0', 1234, opts) do |req|
$throttler.call { req.respond("Hello world!\n") }
end
end
puts "pid: #{Process.pid}"
puts 'Listening on port 1234...'
server.await
================================================
FILE: examples/http_server_throttled_accept.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
::Exception.__disable_sanitized_backtrace__ = true
opts = {
reuse_addr: true,
reuse_port: true,
dont_linger: true
}
server = Tipi.listen('0.0.0.0', 1234, opts)
puts 'Listening on port 1234'
throttler = Polyphony::Throttler.new(interval: 5)
server.accept_loop do |socket|
throttler.call do
spin { Tipi.client_loop(socket, opts) { |req| req.respond("Hello world!\n") } }
end
end
================================================
FILE: examples/http_server_timeout.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
opts = {
reuse_addr: true,
dont_linger: true
}
def timeout_handler(timeout, &handler)
->(req) do
cancel_after(timeout) { handler.(req) }
rescue Polyphony::Cancel
req.respond("timeout\n", ':status' => 502)
end
end
sleep 0
spin do
Tipi.serve(
'0.0.0.0',
1234,
opts,
&timeout_handler(0.1) do |req|
sleep rand(0.01..0.2)
req.respond("Hello timeout world!\n")
end
)
end
puts "pid: #{Process.pid}"
puts 'Listening on port 1234...'
suspend
================================================
FILE: examples/http_unix_socket_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
path = '/tmp/tipi.sock'
puts "pid: #{Process.pid}"
puts "Listening on #{path}"
FileUtils.rm(path) rescue nil
socket = UNIXServer.new(path)
Tipi.accept_loop(socket, {}) do |req|
req.respond("Hello world!\n")
rescue Exception => e
p e
end
================================================
FILE: examples/http_ws_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'tipi/websocket'
def ws_handler(conn)
timer = spin_loop(interval: 1) do
conn << Time.now.to_s
end
while (msg = conn.recv)
conn << "you said: #{msg}"
end
rescue Exception => e
p e
ensure
timer.stop
end
opts = {
reuse_addr: true,
dont_linger: true,
upgrade: {
websocket: Tipi::Websocket.handler(&method(:ws_handler))
}
}
HTML = IO.read(File.join(__dir__, 'ws_page.html'))
puts "pid: #{Process.pid}"
puts 'Listening on port 4411...'
Tipi.serve('0.0.0.0', 4411, opts) do |req|
req.respond(HTML, 'Content-Type' => 'text/html')
end
================================================
FILE: examples/https_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'localhost/authority'
::Exception.__disable_sanitized_backtrace__ = true
authority = Localhost::Authority.fetch
opts = {
reuse_addr: true,
dont_linger: true,
}
puts "pid: #{Process.pid}"
puts 'Listening on port 1234...'
ctx = authority.server_context
server = Polyphony::Net.tcp_listen('0.0.0.0', 1234, opts)
loop do
socket = server.accept
client = OpenSSL::SSL::SSLSocket.new(socket, ctx)
client.sync_close = true
spin do
state = {}
accept_thread = Thread.new do
puts "call client accept"
client.accept
state[:result] = :ok
rescue Exception => e
puts error: e
state[:result] = e
end
"wait for accept thread"
accept_thread.join
"accept thread done"
if state[:result].is_a?(Exception)
puts "Exception in SSL handshake: #{state[:result].inspect}"
next
end
Tipi.client_loop(client, opts) do |req|
p path: req.path
if req.path == '/stream'
req.send_headers('Foo' => 'Bar')
sleep 0.5
req.send_chunk("foo\n")
sleep 0.5
req.send_chunk("bar\n", done: true)
elsif req.path == '/upload'
body = req.read
req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
else
req.respond("Hello world!\n")
end
end
ensure
client ? client.close : socket.close
end
end
================================================
FILE: examples/https_server_forked.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'localhost/authority'
::Exception.__disable_sanitized_backtrace__ = true
authority = Localhost::Authority.fetch
opts = {
reuse_addr: true,
dont_linger: true,
secure_context: authority.server_context
}
server = Tipi.listen('0.0.0.0', 1234, opts)
puts 'Listening on port 1234'
child_pids = []
4.times do
pid = Polyphony.fork do
puts "forked pid: #{Process.pid}"
server.each do |req|
req.respond("Hello world!\n")
end
rescue Interrupt
end
child_pids << pid
end
child_pids.each { |pid| Thread.current.backend.waitpid(pid) }
================================================
FILE: examples/https_wss_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'tipi/websocket'
require 'localhost/authority'
def ws_handler(conn)
timer = spin do
throttled_loop(1) do
conn << Time.now.to_s
rescue StandardError
nil
end
end
while (msg = conn.recv)
puts "msg: #{msg}"
# conn << "you said: #{msg}"
end
ensure
timer.stop
end
authority = Localhost::Authority.fetch
opts = {
reuse_addr: true,
dont_linger: true,
secure_context: authority.server_context,
upgrade: {
websocket: Tipi::Websocket.handler(&method(:ws_handler))
}
}
HTML = IO.read(File.join(__dir__, 'wss_page.html'))
puts "pid: #{Process.pid}"
puts 'Listening on port 1234...'
Tipi.serve('0.0.0.0', 1234, opts) do |req|
req.respond(HTML, 'Content-Type' => 'text/html')
end
================================================
FILE: examples/rack_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
unless File.file?(app_path)
STDERR.puts "Please provide rack config file (there are some in the examples directory.)"
exit!
end
app = Tipi::RackAdapter.load(app_path)
opts = { reuse_addr: true, dont_linger: true }
puts 'listening on port 1234'
puts "pid: #{Process.pid}"
Tipi.serve('0.0.0.0', 1234, opts, &app)
================================================
FILE: examples/rack_server_forked.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
unless File.file?(app_path)
STDERR.puts "Please provide rack config file (there are some in the examples directory.)"
exit!
end
app = Tipi::RackAdapter.load(app_path)
opts = { reuse_addr: true, dont_linger: true }
server = Tipi.listen('0.0.0.0', 1234, opts)
puts 'listening on port 1234'
child_pids = []
4.times do
child_pids << Polyphony.fork do
puts "forked pid: #{Process.pid}"
server.each(&app)
end
end
child_pids.each { |pid| Thread.current.backend.waitpid(pid) }
================================================
FILE: examples/rack_server_https.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'localhost/authority'
app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
app = Tipi::RackAdapter.load(app_path)
authority = Localhost::Authority.fetch
opts = {
reuse_addr: true,
dont_linger: true,
secure_context: authority.server_context
}
puts 'listening on port 1234'
puts "pid: #{Process.pid}"
Tipi.serve('0.0.0.0', 1234, opts, &app)
================================================
FILE: examples/rack_server_https_forked.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'localhost/authority'
app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
app = Tipi::RackAdapter.load(app_path)
authority = Localhost::Authority.fetch
opts = {
reuse_addr: true,
reuse_port: true,
dont_linger: true,
secure_context: authority.server_context
}
server = Tipi.listen('0.0.0.0', 1234, opts)
puts 'Listening on port 1234'
child_pids = []
4.times do
child_pids << Polyphony.fork do
puts "forked pid: #{Process.pid}"
server.each(&app)
end
end
child_pids.each { |pid| Thread.current.backend.waitpid(pid) }
================================================
FILE: examples/routing_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
opts = {
reuse_addr: true,
dont_linger: true
}
puts "pid: #{Process.pid}"
puts 'Listening on port 4411...'
app = Tipi.route do |r|
r.on_root do
r.redirect '/hello'
end
r.on 'hello' do
r.on_get 'world' do
r.respond 'Hello world'
end
r.on_get do
r.respond 'Hello'
end
r.on_post do
puts 'Someone said Hello'
r.redirect '/'
end
end
end
spin do
Tipi.serve('0.0.0.0', 4411, opts, &app)
end.await
================================================
FILE: examples/servername_cb.rb
================================================
# frozen_string_literal: true
require 'openssl'
require 'fiber'
ctx = OpenSSL::SSL::SSLContext.new
f = Fiber.new { |peer| loop { p peer: peer; _name, peer = peer.transfer nil } }
ctx.servername_cb = proc { |_socket, name|
p servername_cb: name
f.transfer([name, Fiber.current]).tap { |r| p result: r }
}
socket = Socket.new(:INET, :STREAM).tap do |s|
s.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1)
s.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
s.bind(Socket.sockaddr_in(12345, '0.0.0.0'))
s.listen(Socket::SOMAXCONN)
end
server = OpenSSL::SSL::SSLServer.new(socket, ctx)
Thread.new do
sleep 0.5
socket = TCPSocket.new('127.0.0.1', 12345)
client = OpenSSL::SSL::SSLSocket.new(socket)
client.hostname = 'example.com'
p client: client
client.connect
rescue => e
p client_error: e
end
while true
conn = server.accept
p accepted: conn
break
end
================================================
FILE: examples/source.rb
================================================
# frozen_string_literal: true
run { |req|
req.serve_file(__FILE__)
}
================================================
FILE: examples/streaming.rb
================================================
# frozen_string_literal: true
run { |req|
req.send_headers('Content-Type' => 'text/event-stream')
10.times { |i|
sleep 0.1
req.send_chunk("data: #{i.to_s * 40}\n")
}
req.finish
}
================================================
FILE: examples/websocket_client.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'polyphony'
require 'websocket'
::Exception.__disable_sanitized_backtrace__ = true
class WebsocketClient
def initialize(url, headers = {})
@socket = TCPSocket.new('127.0.0.1', 1234)
do_handshake(url, headers)
end
def do_handshake(url, headers)
handshake = WebSocket::Handshake::Client.new(url: url, headers: headers)
@socket << handshake.to_s
@socket.read_loop do |data|
handshake << data
break if handshake.finished?
end
raise 'Websocket handshake failed' unless handshake.valid?
@version = handshake.version
@reader = WebSocket::Frame::Incoming::Client.new(version: @version)
end
def receive
@socket.read_loop do |data|
@reader << data
parsed = @reader.next
return parsed if parsed
end
end
def send(data)
frame = WebSocket::Frame::Outgoing::Client.new(
version: @version,
data: data,
type: :text
)
@socket << frame.to_s
end
alias_method :<<, :send
def close
@socket.close
end
end
(1..3).each do |i|
spin do
client = WebsocketClient.new('ws://127.0.0.1:1234/', { Cookie: "SESSIONID=#{i * 10}" })
(1..3).each do |j|
sleep rand(0.2..0.5)
client.send "Hello from client #{i} (#{j})"
puts "server reply: #{client.receive}"
end
client.close
end
end
suspend
================================================
FILE: examples/websocket_demo.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'tipi/websocket'
class WebsocketClient
def initialize(url, headers = {})
@socket = TCPSocket.new('127.0.0.1', 1234)
do_handshake(url, headers)
end
def do_handshake(url, headers)
handshake = WebSocket::Handshake::Client.new(url: url, headers: headers)
@socket << handshake.to_s
@socket.read_loop do |data|
handshake << data
break if handshake.finished?
end
raise 'Websocket handshake failed' unless handshake.valid?
@version = handshake.version
@reader = WebSocket::Frame::Incoming::Client.new(version: @version)
end
def receive
parsed = @reader.next
return parsed if parsed
@socket.read_loop do |data|
@reader << data
parsed = @reader.next
return parsed if parsed
end
end
def send(data)
frame = WebSocket::Frame::Outgoing::Client.new(
version: @version,
data: data,
type: :text
)
@socket << frame.to_s
end
alias_method :<<, :send
def close
@socket.close
end
end
server = spin do
websocket_handler = Tipi::Websocket.handler do |conn|
while (msg = conn.recv)
conn << "you said: #{msg}"
end
end
opts = { upgrade: { websocket: websocket_handler } }
puts 'Listening on port http://127.0.0.1:1234/'
Tipi.serve('0.0.0.0', 1234, opts) do |req|
req.respond("Hello world!\n")
end
end
sleep 0.01 # wait for server to start
clients = (1..3).map do |i|
spin do
client = WebsocketClient.new('ws://127.0.0.1:1234/', { Cookie: "SESSIONID=#{i * 10}" })
(1..3).each do |j|
sleep rand(0.2..0.5)
client.send "Hello from client #{i} (#{j})"
puts "server reply: #{client.receive}"
end
client.close
end
end
Fiber.await(*clients)
================================================
FILE: examples/websocket_secure_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'localhost/authority'
def ws_handler(conn)
while (msg = conn.recv)
conn << "you said: #{msg}"
end
end
authority = Localhost::Authority.fetch
opts = {
reuse_addr: true,
dont_linger: true,
upgrade: {
websocket: Polyphony::Websocket.handler(&method(:ws_handler))
},
secure_context: authority.server_context
}
puts "pid: #{Process.pid}"
puts 'Listening on port 1234...'
Tipi.serve('0.0.0.0', 1234, opts) do |req|
req.respond("Hello world!\n")
end
================================================
FILE: examples/websocket_server.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'tipi'
require 'tipi/websocket'
def ws_handler(conn)
while (msg = conn.recv)
conn << "you said: #{msg}"
end
end
opts = {
reuse_addr: true,
dont_linger: true,
upgrade: {
websocket: Tipi::Websocket.handler(&method(:ws_handler))
}
}
puts "pid: #{Process.pid}"
puts 'Listening on port 1234...'
Tipi.serve('0.0.0.0', 1234, opts) do |req|
req.respond("Hello world!\n")
end
================================================
FILE: examples/ws_page.html
================================================
<!doctype html>
<html lang="en">
<head>
<title>Websocket Client</title>
</head>
<body>
<script>
var connect = function () {
var exampleSocket = new WebSocket("ws://localhost:4411/");
exampleSocket.onopen = function (event) {
document.querySelector('#status').innerText = 'connected';
exampleSocket.send("Can you hear me?");
};
exampleSocket.onclose = function (event) {
document.querySelector('#status').innerText = 'disconnected';
setTimeout(function () {
// exampleSocket.removeAllListeners();
connect();
}, 1000);
}
exampleSocket.onmessage = function (event) {
document.querySelector('#msg').innerText = event.data;
console.log(event.data);
}
};
connect();
</script>
<h1 id="status">disconnected</h1>
<h1 id="msg"></h1>
</body>
</html>
================================================
FILE: examples/wss_page.html
================================================
<!doctype html>
<html lang="en">
<head>
<title>Websocket Client</title>
</head>
<body>
<script>
var connect = function () {
var exampleSocket = new WebSocket("/");
exampleSocket.onopen = function (event) {
document.querySelector('#status').innerText = 'connected';
exampleSocket.send("Can you hear me?");
};
exampleSocket.onclose = function (event) {
console.log('onclose');
document.querySelector('#status').innerText = 'disconnected';
setTimeout(function () {
// exampleSocket.removeAllListeners();
connect();
}, 1000);
}
exampleSocket.onmessage = function (event) {
document.querySelector('#msg').innerText = event.data;
console.log(event.data);
}
};
connect();
</script>
<h1 id="status">disconnected</h1>
<h1 id="msg"></h1>
</body>
</html>
================================================
FILE: examples/zlib-bench.rb
================================================
# frozen_string_literal: true
FILE_SIZES = {
's.tmp' => 2**10,
'm.tmp' => 2**17,
'l.tmp' => 2**20,
'xl.tmp' => 2**24
}
def create_files
FILE_SIZES.each { |fn, size|
IO.write(File.join('/tmp', fn), '*' * size)
}
end
create_files
run { |req|
file_path = File.join('/tmp', req.path)
if File.file?(file_path)
req.serve_file(file_path)
else
req.respond(nil, ':status' => 404)
end
}
================================================
FILE: lib/tipi/acme.rb
================================================
# frozen_string_literal: true
require 'openssl'
require 'acme-client'
require 'localhost/authority'
module Tipi
module ACME
class Error < StandardError
end
class CertificateManager
def initialize(master_ctx:, store:, challenge_handler:, valid_hosts:)
@master_ctx = master_ctx
@store = store
@challenge_handler = challenge_handler
@valid_hosts = valid_hosts
@contexts = {}
@requests = Polyphony::Queue.new
@worker = spin { run }
setup_sni_callback
end
ACME_CHALLENGE_PATH_REGEXP = /\/\.well\-known\/acme\-challenge/.freeze
def challenge_routing_app(app)
->(req) do
(req.path =~ ACME_CHALLENGE_PATH_REGEXP ? @challenge_handler : app)
.(req)
rescue => e
puts "Error while handling request: #{e.inspect} (headers: #{req.headers})"
req.respond(nil, ':status' => Qeweney::Status::BAD_REQUEST)
end
end
IP_REGEXP = /^\d+\.\d+\.\d+\.\d+$/
def setup_sni_callback
@master_ctx.servername_cb = proc { |_socket, name| get_ctx(name) }
end
def get_ctx(name)
state = { ctx: nil }
if @valid_hosts
return nil unless @valid_hosts.include?(name)
end
ready_ctx = @contexts[name]
return ready_ctx if ready_ctx
return @master_ctx if name =~ IP_REGEXP
@requests << [name, state]
wait_for_ctx(state)
# Eventually we might want to return an error returned in
# state[:error]. For the time being we handle errors by returning the
# master context
state[:ctx] || @master_ctx
rescue => e
@master_ctx
end
MAX_WAIT_FOR_CTX_DURATION = 30
def wait_for_ctx(state)
t0 = Time.now
period = 0.00001
while !state[:ctx] && !state[:error]
orig_sleep period
if period < 0.1
period *= 2
elsif Time.now - t0 > MAX_WAIT_FOR_CTX_DURATION
raise "Timeout waiting for certificate provisioning"
end
end
end
def run
loop do
name, state = @requests.shift
state[:ctx] = get_context(name)
rescue => e
state[:error] = e if state
end
end
LOCALHOST_REGEXP = /\.?localhost$/.freeze
def get_context(name)
@contexts[name] = setup_context(name)
end
def setup_context(name)
ctx = provision_context(name)
transfer_ctx_settings(ctx)
ctx
end
def provision_context(name)
return localhost_context if name =~ LOCALHOST_REGEXP
info = get_certificate(name)
ctx = OpenSSL::SSL::SSLContext.new
chain = parse_certificate(info[:certificate])
cert = chain.shift
ctx.add_certificate(cert, info[:private_key], chain)
ctx
end
def transfer_ctx_settings(ctx)
ctx.alpn_protocols = @master_ctx.alpn_protocols
ctx.alpn_select_cb = @master_ctx.alpn_select_cb
ctx.ciphers = @master_ctx.ciphers
end
CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
def parse_certificate(certificate)
certificate
.scan(CERTIFICATE_REGEXP)
.map { |p| OpenSSL::X509::Certificate.new(p.first) }
end
def get_expired_stamp(certificate)
chain = parse_certificate(certificate)
cert = chain.shift
cert.not_after
end
def get_certificate(name)
entry = @store.get(name)
return entry if entry
provision_certificate(name).tap do |entry|
@store.set(name, **entry)
end
end
def localhost_context
@localhost_authority ||= Localhost::Authority.fetch
@localhost_authority.server_context
end
def private_key
@private_key ||= OpenSSL::PKey::RSA.new(4096)
end
ACME_DIRECTORY = 'https://acme-v02.api.letsencrypt.org/directory'
def acme_client
@acme_client ||= setup_acme_client
end
def setup_acme_client
client = Acme::Client.new(
private_key: private_key,
directory: ACME_DIRECTORY
)
account = client.new_account(
contact: 'mailto:info@noteflakes.com',
terms_of_service_agreed: true
)
client
end
def provision_certificate(name)
order = acme_client.new_order(identifiers: [name])
authorization = order.authorizations.first
challenge = authorization.http
@challenge_handler.add(challenge)
challenge.request_validation
while challenge.status == 'pending'
sleep(0.25)
challenge.reload
end
raise ACME::Error, "Invalid CSR" if challenge.status == 'invalid'
private_key = OpenSSL::PKey::RSA.new(4096)
csr = Acme::Client::CertificateRequest.new(
private_key: private_key,
subject: { common_name: name }
)
order.finalize(csr: csr)
while order.status == 'processing'
sleep(0.25)
order.reload
end
certificate = begin
order.certificate(force_chain: 'DST Root CA X3')
rescue Acme::Client::Error::ForcedChainNotFound
order.certificate
end
expired_stamp = get_expired_stamp(certificate)
puts "Certificate for #{name} expires: #{expired_stamp.inspect}"
{
private_key: private_key,
certificate: certificate,
expired_stamp: expired_stamp
}
end
end
class HTTPChallengeHandler
def initialize
@challenges = {}
end
def add(challenge)
path = "/.well-known/acme-challenge/#{challenge.token}"
@challenges[path] = challenge
end
def remove(challenge)
path = "/.well-known/acme-challenge/#{challenge.token}"
@challenges.delete(path)
end
def call(req)
challenge = @challenges[req.path]
# handle incoming request
challenge = @challenges[req.path]
return req.respond(nil, ':status' => 400) unless challenge
req.respond(challenge.file_content, 'content-type' => challenge.content_type)
end
end
class CertificateStore
def set(name, private_key:, certificate:, expired_stamp:)
raise NotImplementedError
end
def get(name)
raise NotImplementedError
end
end
class InMemoryCertificateStore
def initialize
@store = {}
end
def set(name, private_key:, certificate:, expired_stamp:)
@store[name] = {
private_key: private_key,
certificate: certificate,
expired_stamp: expired_stamp
}
end
def get(name)
entry = @store[name]
return nil unless entry
if Time.now >= entry[:expired_stamp]
@store.delete(name)
return nil
end
entry
end
end
class SQLiteCertificateStore
attr_reader :db
def initialize(path)
require 'extralite'
@db = Extralite::Database.new(path)
@db.query("
create table if not exists certificates (
name primary key not null,
private_key not null,
certificate not null,
expired_stamp not null
);"
)
end
def set(name, private_key:, certificate:, expired_stamp:)
@db.query("
insert into certificates values (?, ?, ?, ?)
", name, private_key.to_s, certificate, expired_stamp.to_i)
rescue Extralite::Error => e
p error_in_set: e
raise e
end
def get(name)
remove_expired_certificates
entry = @db.query_single_row("
select name, private_key, certificate, expired_stamp
from certificates
where name = ?
", name)
return nil unless entry
entry[:expired_stamp] = Time.at(entry[:expired_stamp])
entry[:private_key] = OpenSSL::PKey::RSA.new(entry[:private_key])
entry
rescue Extralite::Error => e
p error_in_get: e
raise e
end
def remove_expired_certificates
@db.query("
delete from certificates
where expired_stamp < ?
", Time.now.to_i)
rescue Extralite::Error => e
p error_in_remove_expired_certificates: e
raise e
end
end
end
end
================================================
FILE: lib/tipi/cli.rb
================================================
# frozen_string_literal: true
require 'tipi'
require 'fileutils'
require 'tipi/supervisor'
require 'optparse'
module Tipi
DEFAULT_OPTS = {
app_type: :web,
mode: :polyphony,
workers: 1,
threads: 1,
listen: ['http', 'localhost', 1234],
path: '.',
}
def self.opts_from_argv(argv)
opts = DEFAULT_OPTS.dup
parser = OptionParser.new do |o|
o.banner = "Usage: tipi [options] path"
o.on('-h', '--help', 'Show this help') { puts o; exit }
o.on('-wNUM', '--workers NUM', 'Number of worker processes (default: 1)') do |v|
opts[:workers] = v
end
o.on('-tNUM', '--threads NUM', 'Number of worker threads (default: 1)') do |v|
opts[:threads] = v
opts[:mode] = :stock
end
o.on('-c', '--compatibility', 'Use compatibility mode') do
opts[:mode] = :stock
end
o.on('-lSPEC', '--listen SPEC', 'Setup HTTP listener') do |v|
opts[:listen] = parse_listen_spec('http', v)
end
o.on('-sSPEC', '--secure SPEC', 'Setup HTTPS listener (for localhost)') do |v|
opts[:listen] = parse_listen_spec('https', v)
end
o.on('-fSPEC', '--full-service SPEC', 'Setup HTTP/HTTPS listeners (with automatic certificates)') do |v|
opts[:listen] = parse_listen_spec('full', v)
end
o.on('-v', '--verbose', 'Verbose output') do
opts[:verbose] = true
end
end.parse!(argv)
opts[:path] = argv.shift unless argv.empty?
verify_path(opts[:path])
opts
end
def self.parse_listen_spec(type, spec)
[type, *spec.split(':').map { |s| str_to_native_type(s) }]
end
def self.str_to_native_type(str)
case str
when /^\d+$/
str.to_i
else
str
end
end
def self.verify_path(path)
return if File.file?(path) || File.directory?(path)
puts "Invalid path specified #{path}"
exit!
end
module CLI
BANNER =
"\n" +
" ooo\n" +
" oo\n" +
" o\n" +
" \\|/ Tipi - a better web server for a better world\n" +
" / \\ \n" +
" / \\ https://github.com/digital-fabric/tipi\n" +
"⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺⎺\n"
def self.start(argv = ARGV.dup)
opts = Tipi.opts_from_argv(argv)
display_banner if STDOUT.tty? && !opts[:silent]
Tipi::Supervisor.run(opts)
end
def self.display_banner
puts BANNER
end
end
end
================================================
FILE: lib/tipi/config_dsl.rb
================================================
# frozen_string_literal: true
module Tipi
module Configuration
class Interpreter
# make_blank_slate
def initialize(assembler)
@assembler = assembler
end
def gzip_response
@assembler.emit 'req = Tipi::GZip.wrap(req)'
end
def log(out)
@assembler.wrap_current_frame 'logger.log_request(req) do |req|'
end
def error(&block)
assembler.emit_exception_handler &block
end
def match(pattern, &block)
@assembler.emit_conditional "if req.path =~ #{pattern.inspect}", &block
end
end
class Assembler
def self.from_source(code)
new.from_source code
end
def from_source(code)
@stack = [new_frame]
@app_procs = {}
@interpreter = Interpreter.new self
@interpreter.instance_eval code
loop do
frame = @stack.pop
return assemble_app_proc(frame).join("\n") if @stack.empty?
@stack.last[:body] << assemble_frame(frame)
end
end
def new_frame
{
prelude: [],
body: []
}
end
def add_frame(&block)
@stack.push new_frame
yield
ensure
frame = @stack.pop
emit assemble(frame)
end
def wrap_current_frame(head)
frame = @stack.pop
wrapper = new_frame
wrapper[:body] << head
@stack.push wrapper
@stack.push frame
end
def emit(code)
@stack.last[:body] << code
end
def emit_prelude(code)
@stack.last[:prelude] << code
end
def emit_exception_handler(&block)
proc_id = add_app_proc block
@stack.last[:rescue_proc_id] = proc_id
end
def emit_block(conditional, &block)
proc_id = add_app_proc block
@stack.last[:branched] = true
emit conditional
add_frame &block
end
def add_app_proc(proc)
id = :"proc#{@app_procs.size}"
@app_procs[id] = proc
id
end
def assemble_frame(frame)
indent = 0
lines = []
emit_code lines, frame[:prelude], indent
if frame[:rescue_proc_id]
emit_code lines, 'begin', indent
indent += 1
end
emit_code lines, frame[:body], indent
if frame[:rescue_proc_id]
emit_code lines, 'rescue => e', indent
emit_code lines, " app_procs[#{frame[:rescue_proc_id].inspect}].call(req, e)", indent
emit_code lines, 'end', indent
indent -= 1
end
lines
end
def assemble_app_proc(frame)
indent = 0
lines = []
emit_code lines, frame[:prelude], indent
emit_code lines, 'proc do |req|', indent
emit_code lines, frame[:body], indent + 1
emit_code lines, 'end', indent
lines
end
def emit_code(lines, code, indent)
if code.is_a? Array
code.each { |l| emit_code lines, l, indent + 1 }
else
lines << (indent_line code, indent)
end
end
@@indents = Hash.new { |h, k| h[k] = ' ' * k }
def indent_line(code, indent)
indent == 0 ? code : "#{ @@indents[indent] }#{code}"
end
end
end
end
def assemble(code)
Tipi::Configuration::Assembler.from_source(code)
end
code = assemble <<~RUBY
gzip_response
log STDOUT
RUBY
puts code
================================================
FILE: lib/tipi/configuration.rb
================================================
# frozen_string_literal: true
require_relative './handler'
module Tipi
module Configuration
class << self
def supervise_config
current_runner = nil
while (config = receive)
old_runner, current_runner = current_runner, spin { run(config) }
old_runner&.stop
end
end
def run(config)
start_listeners(config)
config[:forked] ? forked_supervise(config) : simple_supervise(config)
end
def start_listeners(config)
puts "Listening on port 1234"
@server = Polyphony::Net.tcp_listen('0.0.0.0', 1234, { reuse_addr: true, dont_linger: true })
end
def simple_supervise(config)
virtual_hosts = setup_virtual_hosts(config)
start_acceptors(config, virtual_hosts)
suspend
# supervise(restart: true)
end
def forked_supervise(config)
config[:forked].times do
spin { Polyphony.watch_process { simple_supervise(config) } }
end
suspend
end
def setup_virtual_hosts(config)
{
'*': Tipi::DefaultHandler.new(config)
}
end
def start_acceptors(config, virtual_hosts)
spin do
puts "pid: #{Process.pid}"
while (connection = @server.accept)
spin { virtual_hosts[:'*'].call(connection) }
end
end
end
end
end
end
================================================
FILE: lib/tipi/controller/bare_polyphony.rb
================================================
================================================
FILE: lib/tipi/controller/bare_stock.rb
================================================
# frozen_string_literal: true
module Tipi
module Apps
module Bare
def start(opts)
end
end
end
end
================================================
FILE: lib/tipi/controller/extensions.rb
================================================
# frozen_string_literal: true
require 'tipi'
module Kernel
def run(app = nil, &block)
Tipi.app = app || block
end
end
module Tipi
class << self
attr_writer :app
def app
return @app if @app
raise 'No app define. The app to run should be set using `Tipi.app = ...`'
end
def run_sites(site_map)
sites = site_map.each_with_object({}) { |(k, v), h| h[k] = v.to_proc }
valid_hosts = sites.keys
@app = ->(req) {
handler = sites[req.host]
if handler
handler.call(req)
else
req.respond(nil, ':status' => Qeweney::Status::NOT_FOUND)
end
}
@app.define_singleton_method(:valid_hosts) { valid_hosts }
end
end
end
================================================
FILE: lib/tipi/controller/stock_http1_adapter.rb
================================================
# frozen_string_literal: true
require 'tipi/http1_adapter'
module Tipi
class StockHTTP1Adapter < HTTP1Adapter
def initialize(conn, opts)
super(conn, opts)
end
def each(&block)
end
end
end
================================================
FILE: lib/tipi/controller/web_polyphony.rb
================================================
# frozen_string_literal: true
require 'tipi'
require 'localhost/authority'
require_relative './extensions'
module Tipi
class Controller
def initialize(opts)
@opts = opts
@path = File.expand_path(@opts['path'])
@service = prepare_service
end
WORKER_COUNT_RANGE = (1..32).freeze
def run
worker_count = (@opts['workers'] || 1).to_i.clamp(WORKER_COUNT_RANGE)
return run_worker if worker_count == 1
supervise_workers(worker_count)
end
private
def supervise_workers(worker_count)
supervisor = spin(:web_worker_supervisor) do
worker_count.times do
spin(:web_worker) do
pid = Polyphony.fork { run_worker }
puts "Forked worker pid: #{pid}"
Polyphony.backend_waitpid(pid)
puts "Done worker pid: #{pid}"
end
end
supervise(restart: :always)
rescue Polyphony::Terminate
# TODO: find out how Terminate can leak like that (it's supposed to be
# caught in Fiber#run)
end
trap('SIGTERM') { supervisor.terminate(graceful: true) }
trap('SIGINT') do
trap('SIGINT') { exit! }
supervisor.terminate(graceful: true)
end
supervisor.await
rescue Polyphony::Terminate
# TODO: find out how Terminate can leak etc.
end
def run_worker
server = start_server(@service)
trap('SIGTERM') { server&.terminate(graceful: true) }
trap('SIGINT') do
trap('SIGINT') { exit! }
server&.terminate(graceful: true)
end
raise 'Server not started' unless server
server.await
rescue Polyphony::Terminate
# TODO: find out why this exception leaks from the server fiber
# ignore
end
def prepare_service
if File.file?(@path)
File.extname(@path) == '.ru' ? rack_service : tipi_service
elsif File.directory?(@path)
static_service
else
raise "Invalid path specified #{@path}"
end
end
def start_app
if File.extname(@path) == '.ru'
start_rack_app
else
require(@path)
end
end
def rack_service
puts "Loading Rack app from #{@path}"
app = Tipi::RackAdapter.load(@path)
web_service(app)
end
def tipi_service
puts "Loading Tipi app from #{@path}"
require(@path)
app = Tipi.app
web_service(app)
# proc { spin { Object.run } }
end
def static_service
puts "Serving static files from #{@path}"
app = proc do |req|
full_path = find_path(@path, req.path)
if full_path
req.serve_file(full_path)
else
req.respond(nil, ':status' => Qeweney::Status::NOT_FOUND)
end
end
web_service(app)
end
def web_service(app)
app = add_connection_headers(app)
prepare_listener(@opts['listen'], app)
end
def prepare_listener(spec, app)
case spec.shift
when 'http'
case spec.size
when 2
host, port = spec
port ||= 80
when 1
host = '0.0.0.0'
port = spec.first || 80
else
raise "Invalid listener spec"
end
prepare_http_listener(port, app)
when 'https'
case spec.size
when 2
host, port = spec
port ||= 80
when 1
host = 'localhost'
port = spec.first || 80
else
raise "Invalid listener spec"
end
port ||= 443
prepare_https_listener(host, port, app)
when 'full'
host, http_port, https_port = spec
http_port ||= 80
https_port ||= 443
prepare_full_service_listeners(host, http_port, https_port, app)
end
end
def prepare_http_listener(port, app)
puts "Listening for HTTP on localhost:#{port}"
proc do
spin_accept_loop('HTTP', port) do |socket|
Tipi.client_loop(socket, @opts, &app)
end
end
end
LOCALHOST_REGEXP = /^(.+\.)?localhost$/.freeze
def prepare_https_listener(host, port, app)
localhost = host =~ LOCALHOST_REGEXP
return prepare_localhost_https_listener(port, app) if localhost
raise "No certificate found for #{host}"
# TODO: implement loading certificate
end
def prepare_localhost_https_listener(port, app)
puts "Listening for HTTPS on localhost:#{port}"
authority = Localhost::Authority.fetch
ctx = authority.server_context
ctx.ciphers = 'ECDH+aRSA'
Polyphony::Net.setup_alpn(ctx, Tipi::ALPN_PROTOCOLS)
proc do
https_listener = spin_accept_loop('HTTPS', port) do |socket|
start_https_connection_fiber(socket, ctx, nil, app)
rescue Exception => e
puts "Exception in https_listener block: #{e.inspect}\n#{e.backtrace.inspect}"
end
end
end
def prepare_full_service_listeners(host, http_port, https_port, app)
puts "Listening for HTTP on localhost:#{http_port}"
puts "Listening for HTTPS on localhost:#{https_port}"
redirect_app = http_redirect_app(https_port)
ctx = OpenSSL::SSL::SSLContext.new
ctx.ciphers = 'ECDH+aRSA'
Polyphony::Net.setup_alpn(ctx, Tipi::ALPN_PROTOCOLS)
certificate_store = create_certificate_store
proc do
challenge_handler = Tipi::ACME::HTTPChallengeHandler.new
certificate_manager = Tipi::ACME::CertificateManager.new(
master_ctx: ctx,
store: certificate_store,
challenge_handler: challenge_handler,
valid_hosts: app.respond_to?(:valid_hosts) ? app.valid_hosts : nil
)
http_app = certificate_manager.challenge_routing_app(redirect_app)
http_listener = spin_accept_loop('HTTP', http_port) do |socket|
Tipi.client_loop(socket, @opts, &http_app)
end
ssl_accept_thread_pool = Polyphony::ThreadPool.new(4)
https_listener = spin_accept_loop('HTTPS', https_port) do |socket|
start_https_connection_fiber(socket, ctx, ssl_accept_thread_pool, app)
rescue Exception => e
puts "Exception in https_listener block: #{e.inspect}\n#{e.backtrace.inspect}"
end
end
end
def http_redirect_app(https_port)
case https_port
when 443, 10443
->(req) { req.redirect("https://#{req.host}#{req.path}") }
else
->(req) { req.redirect("https://#{req.host}:#{https_port}#{req.path}") }
end
end
INVALID_PATH_REGEXP = /\/?(\.\.|\.)\//
def find_path(base, path)
return nil if path =~ INVALID_PATH_REGEXP
full_path = File.join(base, path)
return full_path if File.file?(full_path)
return find_path(full_path, 'index') if File.directory?(full_path)
qualified = "#{full_path}.html"
return qualified if File.file?(qualified)
nil
end
SOCKET_OPTS = {
reuse_addr: true,
reuse_port: true,
dont_linger: true,
}.freeze
def spin_accept_loop(name, port, &block)
spin(:accept_loop) do
server = Polyphony::Net.tcp_listen('0.0.0.0', port, SOCKET_OPTS)
loop do
socket = server.accept
spin_connection_handler(name, socket, block)
rescue Polyphony::BaseException => e
raise
rescue Exception => e
puts "#{name} listener uncaught exception: #{e.inspect}"
end
ensure
finalize_listener(server) if server
end
end
def spin_connection_handler(name, socket, block)
spin(:connection_handler) do
block.(socket)
rescue Polyphony::BaseException
raise
rescue Exception => e
puts "Uncaught error in #{name} handler: #{e.inspect}"
p e.backtrace
end
end
def finalize_listener(server)
fiber = Fiber.current
server.close
gracefully_terminate_conections(fiber) if fiber.graceful_shutdown?
rescue Polyphony::BaseException
raise
rescue Exception => e
trace "Exception in finalize_listener: #{e.inspect}"
end
def gracefully_terminate_conections(fiber)
supervisor = spin(:connection_termination_supervisor) { supervise }.detach
fiber.attach_all_children_to(supervisor)
# terminating the supervisor will
supervisor.terminate(graceful: true)
end
def add_connection_headers(app)
app
# proc do |req|
# conn = req.adapter.conn
# # req.headers[':peer'] = conn.peeraddr(false)[2]
# req.headers[':scheme'] ||= conn.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
# app.(req)
# end
end
def ssl_accept(client)
client.accept
true
rescue Polyphony::BaseException
raise
rescue Exception => e
p e
e
end
def start_https_connection_fiber(socket, ctx, thread_pool, app)
client = OpenSSL::SSL::SSLSocket.new(socket, ctx)
client.sync_close = true
result = thread_pool ?
thread_pool.process { ssl_accept(client) } : ssl_accept(client)
if result.is_a?(Exception)
puts "Exception in SSL handshake: #{result.inspect}"
return
end
Tipi.client_loop(client, @opts, &app)
rescue => e
puts "Uncaught error in HTTPS connection fiber: #{e.inspect} bt: #{e.backtrace.inspect}"
ensure
(client ? client.close : socket.close) rescue nil
end
CERTIFICATE_STORE_DEFAULT_DIR = File.expand_path('~/.tipi').freeze
CERTIFICATE_STORE_DEFAULT_DB_PATH = File.join(
CERTIFICATE_STORE_DEFAULT_DIR, 'certificates.db').freeze
def create_certificate_store
FileUtils.mkdir(CERTIFICATE_STORE_DEFAULT_DIR) rescue nil
Tipi::ACME::SQLiteCertificateStore.new(CERTIFICATE_STORE_DEFAULT_DB_PATH)
end
def start_server(service)
spin(:web_server) do
service.call
supervise(restart: :always)
end
end
end
end
================================================
FILE: lib/tipi/controller/web_stock.rb
================================================
# frozen_string_literal: true
require 'ever'
require 'localhost/authority'
require 'http/parser'
require 'qeweney'
require 'tipi/rack_adapter'
require_relative './extensions'
module Tipi
class Listener
def initialize(server, &handler)
@server = server
@handler = handler
end
def accept
socket, _addrinfo = @server.accept
@handler.call(socket)
end
end
class Connection
def io_ready
raise NotImplementedError
end
end
class HTTP1Connection < Connection
attr_reader :io
def initialize(io, evloop, &app)
@io = io
@evloop = evloop
@parser = Http::Parser.new(self)
@app = app
setup_read_request
end
def setup_read_request
@request_complete = nil
@request = nil
@response_buffer = nil
end
def on_headers_complete(headers)
headers = normalize_headers(headers)
headers[':path'] = @parser.request_url
headers[':method'] = @parser.http_method.downcase
scheme = (proto = headers['x-forwarded-proto']) ?
proto.downcase : scheme_from_connection
headers[':scheme'] = scheme
@request = Qeweney::Request.new(headers, self)
end
def normalize_headers(headers)
headers.each_with_object({}) do |(k, v), h|
k = k.downcase
hk = h[k]
if hk
hk = h[k] = [hk] unless hk.is_a?(Array)
v.is_a?(Array) ? hk.concat(v) : hk << v
else
h[k] = v
end
end
end
def scheme_from_connection
@io.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
end
def on_body(chunk)
@request.buffer_body_chunk(chunk)
end
def on_message_complete
@request_complete = true
end
def io_ready
if !@request_complete
handle_read_request
else
handle_write_response
end
end
def handle_read_request
result = @io.read_nonblock(16384, exception: false)
case result
when :wait_readable
watch_io(false)
when :wait_writable
watch_io(true)
when nil
close_io
else
@parser << result
if @request_complete
handle_request
# @response = handle_request(@request_headers, @request_body)
# handle_write_response
else
watch_io(false)
end
end
rescue HTTP::Parser::Error, SystemCallError, IOError
close_io
end
def watch_io(rw)
@evloop.watch_io(self, @io, rw, true)
# @evloop.emit([:watch_io, self, @io, rw, true])
end
def close_io
@evloop.emit([:close_io, self, @io])
end
def handle_request
@app.call(@request)
# req = Qeweney::Request.new(headers, self)
# response_body = "Hello, world!"
# "HTTP/1.1 200 OK\nContent-Length: #{response_body.bytesize}\n\n#{response_body}"
end
# response API
CRLF = "\r\n"
CRLF_ZERO_CRLF_CRLF = "\r\n0\r\n\r\n"
# Sends response including headers and body. Waits for the request to complete
# if not yet completed. The body is sent using chunked transfer encoding.
# @param request [Qeweney::Request] HTTP request
# @param body [String] response body
# @param headers
def respond(request, body, headers)
formatted_headers = format_headers(headers, body, false)
request.tx_incr(formatted_headers.bytesize + (body ? body.bytesize : 0))
if body
handle_write(formatted_headers + body)
else
handle_write(formatted_headers)
end
end
# Sends response headers. If empty_response is truthy, the response status
# code will default to 204, otherwise to 200.
# @param request [Qeweney::Request] HTTP request
# @param headers [Hash] response headers
# @param empty_response [boolean] whether a response body will be sent
# @param chunked [boolean] whether to use chunked transfer encoding
# @return [void]
def send_headers(request, headers, empty_response: false, chunked: true)
formatted_headers = format_headers(headers, !empty_response, http1_1?(request) && chunked)
request.tx_incr(formatted_headers.bytesize)
handle_write(formatted_headers)
end
def http1_1?(request)
request.headers[':protocol'] == 'http/1.1'
end
# Sends a response body chunk. If no headers were sent, default headers are
# sent using #send_headers. if the done option is true(thy), an empty chunk
# will be sent to signal response completion to the client.
# @param request [Qeweney::Request] HTTP request
# @param chunk [String] response body chunk
# @param done [boolean] whether the response is completed
# @return [void]
def send_chunk(request, chunk, done: false)
data = +''
data << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n" if chunk
data << "0\r\n\r\n" if done
return if data.empty?
request.tx_incr(data.bytesize)
handle_write(data)
end
# Finishes the response to the current request. If no headers were sent,
# default headers are sent using #send_headers.
# @return [void]
def finish(request)
request.tx_incr(5)
handle_write("0\r\n\r\n")
end
INTERNAL_HEADER_REGEXP = /^:/.freeze
# Formats response headers into an array. If empty_response is true(thy),
# the response status code will default to 204, otherwise to 200.
# @param headers [Hash] response headers
# @param body [boolean] whether a response body will be sent
# @param chunked [boolean] whether to use chunked transfer encoding
# @return [String] formatted response headers
def format_headers(headers, body, chunked)
status = headers[':status']
status ||= (body ? Qeweney::Status::OK : Qeweney::Status::NO_CONTENT)
lines = format_status_line(body, status, chunked)
headers.each do |k, v|
next if k =~ INTERNAL_HEADER_REGEXP
collect_header_lines(lines, k, v)
end
lines << CRLF
lines
end
def format_status_line(body, status, chunked)
if !body
empty_status_line(status)
else
with_body_status_line(status, body, chunked)
end
end
def empty_status_line(status)
if status == 204
+"HTTP/1.1 #{status}\r\n"
else
+"HTTP/1.1 #{status}\r\nContent-Length: 0\r\n"
end
end
def with_body_status_line(status, body, chunked)
if chunked
+"HTTP/1.1 #{status}\r\nTransfer-Encoding: chunked\r\n"
else
+"HTTP/1.1 #{status}\r\nContent-Length: #{body.is_a?(String) ? body.bytesize : body.to_i}\r\n"
end
end
def collect_header_lines(lines, key, value)
if value.is_a?(Array)
value.inject(lines) { |_, item| lines << "#{key}: #{item}\r\n" }
else
lines << "#{key}: #{value}\r\n"
end
end
def handle_write(data = nil)
if data
if @response_buffer
@response_buffer << data
else
@response_buffer = +data
end
end
result = @io.write_nonblock(@response_buffer, exception: false)
case result
when :wait_readable
watch_io(false)
when :wait_writable
watch_io(true)
when nil
close_io
else
setup_read_request
watch_io(false)
end
end
end
class Controller
def initialize(opts)
@opts = opts
@path = File.expand_path(@opts['path'])
@service = prepare_service
end
WORKER_COUNT_RANGE = (1..32).freeze
def run
worker_count = (@opts['workers'] || 1).to_i.clamp(WORKER_COUNT_RANGE)
return run_worker if worker_count == 1
supervise_workers(worker_count)
end
private
def supervise_workers(worker_count)
supervisor = spin do
worker_count.times do
pid = fork { run_worker }
puts "Forked worker pid: #{pid}"
Process.wait(pid)
puts "Done worker pid: #{pid}"
end
# supervise(restart: :always)
rescue Polyphony::Terminate
# TODO: find out how Terminate can leak like that (it's supposed to be
# caught in Fiber#run)
end
# trap('SIGTERM') { supervisor.terminate(graceful: true) }
# trap('SIGINT') do
# trap('SIGINT') { exit! }
# supervisor.terminate(graceful: true)
# end
# supervisor.await
end
def run_worker
@evloop = Ever::Loop.new
start_server(@service)
trap('SIGTERM') { @evloop.stop }
trap('SIGINT') do
trap('SIGINT') { exit! }
@evloop.stop
end
run_evloop
end
def run_evloop
@evloop.each do |event|
case event
when Listener
event.accept
when Connection
event.io_ready
when Array
cmd, key, io, rw, oneshot = event
case cmd
when :watch_io
@evloop.watch_io(key, io, rw, oneshot)
when :close_io
io.close
end
end
end
end
def prepare_service
if File.file?(@path)
File.extname(@path) == '.ru' ? rack_service : tipi_service
elsif File.directory?(@path)
static_service
else
raise "Invalid path specified #{@path}"
end
end
def start_app
if File.extname(@path) == '.ru'
start_rack_app
else
require(@path)
end
end
def rack_service
puts "Loading Rack app from #{@path}"
app = Tipi::RackAdapter.load(@path)
web_service(app)
end
def tipi_service
puts "Loading Tipi app from #{@path}"
require(@path)
app = Tipi.app
if !app
raise "No app define. The app to run should be set using `Tipi.app = ...`"
end
web_service(app)
end
def static_service
puts "Serving static files from #{@path}"
app = proc do |req|
p req: req
full_path = find_path(@path, req.path)
if full_path
req.serve_file(full_path)
else
req.respond(nil, ':status' => Qeweney::Status::NOT_FOUND)
end
end
web_service(app)
end
def web_service(app)
app = add_connection_headers(app)
prepare_listener(@opts['listen'], app)
end
def prepare_listener(spec, app)
case spec.shift
when 'http'
case spec.size
when 2
host, port = spec
port ||= 80
when 1
host = '0.0.0.0'
port = spec.first || 80
else
raise "Invalid listener spec"
end
prepare_http_listener(port, app)
when 'https'
case spec.size
when 2
host, port = spec
port ||= 80
when 1
host = 'localhost'
port = spec.first || 80
else
raise "Invalid listener spec"
end
port ||= 443
prepare_https_listener(host, port, app)
when 'full'
host, http_port, https_port = spec
http_port ||= 80
https_port ||= 443
prepare_full_service_listeners(host, http_port, https_port, app)
end
end
def prepare_http_listener(port, app)
puts "Listening for HTTP on localhost:#{port}"
proc do
start_listener('HTTP', port) do |socket|
start_client(socket, &app)
end
end
end
def start_client(socket, &app)
conn = HTTP1Connection.new(socket, @evloop, &app)
conn.watch_io(false)
end
LOCALHOST_REGEXP = /^(.+\.)?localhost$/.freeze
def prepare_https_listener(host, port, app)
localhost = host =~ LOCALHOST_REGEXP
return prepare_localhost_https_listener(port, app) if localhost
raise "No certificate found for #{host}"
# TODO: implement loading certificate
end
def prepare_localhost_https_listener(port, app)
puts "Listening for HTTPS on localhost:#{port}"
authority = Localhost::Authority.fetch
ctx = authority.server_context
ctx.ciphers = 'ECDH+aRSA'
Polyphony::Net.setup_alpn(ctx, Tipi::ALPN_PROTOCOLS)
proc do
https_listener = spin_accept_loop('HTTPS', port) do |socket|
start_https_connection_fiber(socket, ctx, nil, app)
rescue Exception => e
puts "Exception in https_listener block: #{e.inspect}\n#{e.backtrace.inspect}"
end
end
end
def prepare_full_service_listeners(host, http_port, https_port, app)
puts "Listening for HTTP on localhost:#{http_port}"
puts "Listening for HTTPS on localhost:#{https_port}"
redirect_host = (https_port == 443) ? host : "#{host}:#{https_port}"
redirect_app = ->(r) { r.redirect("https://#{redirect_host}#{r.path}") }
ctx = OpenSSL::SSL::SSLContext.new
ctx.ciphers = 'ECDH+aRSA'
Polyphony::Net.setup_alpn(ctx, Tipi::ALPN_PROTOCOLS)
certificate_store = create_certificate_store
proc do
challenge_handler = Tipi::ACME::HTTPChallengeHandler.new
certificate_manager = Tipi::ACME::CertificateManager.new(
master_ctx: ctx,
store: certificate_store,
challenge_handler: challenge_handler
)
http_app = certificate_manager.challenge_routing_app(redirect_app)
http_listener = spin_accept_loop('HTTP', http_port) do |socket|
Tipi.client_loop(socket, @opts, &http_app)
end
ssl_accept_thread_pool = Polyphony::ThreadPool.new(4)
https_listener = spin_accept_loop('HTTPS', https_port) do |socket|
start_https_connection_fiber(socket, ctx, ssl_accept_thread_pool, app)
rescue Exception => e
puts "Exception in https_listener block: #{e.inspect}\n#{e.backtrace.inspect}"
end
end
end
INVALID_PATH_REGEXP = /\/?(\.\.|\.)\//
def find_path(base, path)
return nil if path =~ INVALID_PATH_REGEXP
full_path = File.join(base, path)
return full_path if File.file?(full_path)
return find_path(full_path, 'index') if File.directory?(full_path)
qualified = "#{full_path}.html"
return qualified if File.file?(qualified)
nil
end
SOCKET_OPTS = {
reuse_addr: true,
reuse_port: true,
dont_linger: true,
}.freeze
def start_listener(name, port, &block)
host = '0.0.0.0'
socket = ::Socket.new(:INET, :STREAM).tap do |s|
s.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1)
s.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
s.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, [0, 0].pack('ii'))
addr = ::Socket.sockaddr_in(port, host)
s.bind(addr)
s.listen(Socket::SOMAXCONN)
end
listener = Listener.new(socket, &block)
@evloop.watch_io(listener, socket, false, false)
end
def spin_accept_loop(name, port, &block)
spin do
server = Polyphony::Net.tcp_listen('0.0.0.0', port, SOCKET_OPTS)
loop do
socket = server.accept
spin_connection_handler(name, socket, block)
rescue Polyphony::BaseException => e
raise
rescue Exception => e
puts "#{name} listener uncaught exception: #{e.inspect}"
end
ensure
finalize_listener(server) if server
end
end
def spin_connection_handler(name, socket, block)
spin do
block.(socket)
rescue Polyphony::BaseException
raise
rescue Exception => e
puts "Uncaught error in #{name} handler: #{e.inspect}"
p e.backtrace
end
end
def finalize_listener(server)
fiber = Fiber.current
gracefully_terminate_conections(fiber) if fiber.graceful_shutdown?
server.close
rescue Polyphony::BaseException
raise
rescue Exception => e
trace "Exception in finalize_listener: #{e.inspect}"
end
def gracefully_terminate_conections(fiber)
supervisor = spin { supervise }.detach
fiber.attach_all_children_to(supervisor)
# terminating the supervisor will
supervisor.terminate(graceful: true)
end
def add_connection_headers(app)
app
# proc do |req|
# conn = req.adapter.conn
# # req.headers[':peer'] = conn.peeraddr(false)[2]
# req.headers[':scheme'] ||= conn.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
# app.(req)
# end
end
def ssl_accept(client)
client.accept
true
rescue Polyphony::BaseException
raise
rescue Exception => e
p e
e
end
def start_https_connection_fiber(socket, ctx, thread_pool, app)
client = OpenSSL::SSL::SSLSocket.new(socket, ctx)
client.sync_close = true
result = thread_pool ?
thread_pool.process { ssl_accept(client) } : ssl_accept(client)
if result.is_a?(Exception)
puts "Exception in SSL handshake: #{result.inspect}"
return
end
Tipi.client_loop(client, @opts, &app)
rescue => e
puts "Uncaught error in HTTPS connection fiber: #{e.inspect} bt: #{e.backtrace.inspect}"
ensure
(client ? client.close : socket.close) rescue nil
end
CERTIFICATE_STORE_DEFAULT_DIR = File.expand_path('~/.tipi').freeze
CERTIFICATE_STORE_DEFAULT_DB_PATH = File.join(
CERTIFICATE_STORE_DEFAULT_DIR, 'certificates.db').freeze
def create_certificate_store
FileUtils.mkdir(CERTIFICATE_STORE_DEFAULT_DIR) rescue nil
Tipi::ACME::SQLiteCertificateStore.new(CERTIFICATE_STORE_DEFAULT_DB_PATH)
end
def start_server(service)
service.call
end
end
end
================================================
FILE: lib/tipi/controller.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'json'
# get opts from STDIN
opts = JSON.parse(ARGV[0]) rescue nil
mod_path = "./controller/#{opts['app_type']}_#{opts['mode']}"
require_relative mod_path
controller = Tipi::Controller.new(opts)
controller.run
================================================
FILE: lib/tipi/digital_fabric/agent.rb
================================================
# frozen_string_literal: true
require_relative './protocol'
require_relative './request_adapter'
require 'msgpack'
require 'tipi/websocket'
require 'tipi/request'
module DigitalFabric
class Agent
def initialize(server_url, route, token)
@server_url = server_url
@route = route
@token = token
@requests = {}
@long_running_requests = {}
@name = '<unknown>'
end
class TimeoutError < RuntimeError
end
class GracefulShutdown < RuntimeError
end
@@id = 0
def run
@fiber = Fiber.current
@keep_alive_timer = spin_loop("#{@fiber.tag}-keep_alive", interval: 5) { keep_alive }
while true
connect_and_process_incoming_requests
return if @shutdown
sleep 5
end
ensure
@keep_alive_timer.stop
end
def connect_and_process_incoming_requests
# log 'Connecting...'
@socket = connect_to_server
@last_recv = @last_send = Time.now
df_upgrade
@connected = true
@msgpack_reader = MessagePack::Unpacker.new
process_incoming_requests
rescue IOError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, TimeoutError
log 'Disconnected' if @connected
@connected = nil
end
def connect_to_server
if @server_url =~ /^([^\:]+)\:(\d+)$/
host = Regexp.last_match(1)
port = Regexp.last_match(2)
Polyphony::Net.tcp_connect(host, port)
else
UNIXSocket.new(@server_url)
end
end
UPGRADE_REQUEST = <<~HTTP
GET / HTTP/1.1
Host: localhost
Connection: upgrade
Upgrade: df
DF-Token: %s
DF-Mount: %s
HTTP
def df_upgrade
@socket << format(UPGRADE_REQUEST, @token, mount_point)
while (line = @socket.gets)
break if line.chomp.empty?
end
# log 'Connection upgraded'
end
def mount_point
if @route[:host]
"host=#{@route[:host]}"
elsif @route[:path]
"path=#{@route[:path]}"
else
nil
end
end
def log(msg)
puts "#{Time.now} (#{@name}) #{msg}"
end
def process_incoming_requests
@socket.feed_loop(@msgpack_reader, :feed_each) do |msg|
recv_df_message(msg)
return if @shutdown && @requests.empty?
end
rescue IOError, SystemCallError, TimeoutError
# ignore
end
def keep_alive
return unless @connected
now = Time.now
if now - @last_send >= Protocol::SEND_TIMEOUT
send_df_message(Protocol.ping)
end
# if now - @last_recv >= Protocol::RECV_TIMEOUT
# raise TimeoutError
# end
rescue IOError, SystemCallError => e
# transmit exception to fiber running the agent
@fiber.raise(e)
end
def recv_df_message(msg)
@last_recv = Time.now
case msg[Protocol::Attribute::KIND]
when Protocol::SHUTDOWN
recv_shutdown
when Protocol::HTTP_REQUEST
recv_http_request(msg)
when Protocol::HTTP_REQUEST_BODY
recv_http_request_body(msg)
when Protocol::WS_REQUEST
recv_ws_request(msg)
when Protocol::CONN_DATA, Protocol::CONN_CLOSE,
Protocol::WS_DATA, Protocol::WS_CLOSE
fiber = @requests[msg[Protocol::Attribute::ID]]
fiber << msg if fiber
end
end
def send_df_message(msg)
# we mark long-running requests by applying simple heuristics to sent DF
# messages. This is so we can correctly stop long-running requests
# upon graceful shutdown
if is_long_running_request_response?(msg)
id = msg[Protocol::Attribute::ID]
@long_running_requests[id] = @requests[id]
end
@last_send = Time.now
@socket << msg.to_msgpack
end
def is_long_running_request_response?(msg)
case msg[Protocol::Attribute::KIND]
when Protocol::HTTP_UPGRADE
true
when Protocol::HTTP_RESPONSE
!msg[Protocol::Attribute::HttpResponse::COMPLETE]
end
end
def recv_shutdown
# puts "Received shutdown message (#{@requests.size} pending requests)"
# puts " (Long running requests: #{@long_running_requests.size})"
@shutdown = true
@long_running_requests.values.each { |f| f.terminate(graceful: true) }
end
def recv_http_request(msg)
req = prepare_http_request(msg)
id = msg[Protocol::Attribute::ID]
@requests[id] = spin("#{Fiber.current.tag}.#{id}") do
http_request(req)
rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
# ignore
rescue Polyphony::Terminate => e
req.respond(nil, { ':status' => Qeweney::Status::SERVICE_UNAVAILABLE }) if Fiber.current.graceful_shutdown?
raise e
ensure
@requests.delete(id)
@long_running_requests.delete(id)
@fiber.terminate if @shutdown && @requests.empty?
end
end
def prepare_http_request(msg)
headers = msg[Protocol::Attribute::HttpRequest::HEADERS]
body_chunk = msg[Protocol::Attribute::HttpRequest::BODY_CHUNK]
complete = msg[Protocol::Attribute::HttpRequest::COMPLETE]
req = Qeweney::Request.new(headers, RequestAdapter.new(self, msg))
req.buffer_body_chunk(body_chunk) if body_chunk
req
end
def recv_http_request_body(msg)
fiber = @requests[msg[Protocol::Attribute::ID]]
return unless fiber
fiber << msg[Protocol::Attribute::HttpRequestBody::BODY]
end
def get_http_request_body(id, limit)
send_df_message(Protocol.http_get_request_body(id, limit))
receive
end
def recv_ws_request(msg)
req = Qeweney::Request.new(msg[Protocol::Attribute::WS::HEADERS], RequestAdapter.new(self, msg))
id = msg[Protocol::Attribute::ID]
@requests[id] = @long_running_requests[id] = spin("#{Fiber.current.tag}.#{id}-ws") do
ws_request(req)
rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
# ignore
ensure
@requests.delete(id)
@long_running_requests.delete(id)
@fiber.terminate if @shutdown && @requests.empty?
end
end
# default handler for HTTP request
def http_request(req)
req.respond(nil, { ':status': Qeweney::Status::SERVICE_UNAVAILABLE })
end
# default handler for WS request
def ws_request(req)
req.respond(nil, { ':status': Qeweney::Status::SERVICE_UNAVAILABLE })
end
end
end
================================================
FILE: lib/tipi/digital_fabric/agent_proxy.rb
================================================
# frozen_string_literal: true
require_relative './protocol'
require 'msgpack'
require 'tipi/websocket'
module DigitalFabric
class AgentProxy
def initialize(service, req)
@service = service
@req = req
@conn = req.adapter.conn
@msgpack_reader = MessagePack::Unpacker.new
@requests = {}
@current_request_count = 0
@last_request_id = 0
@last_recv = @last_send = Time.now
run
end
def current_request_count
@current_request_count
end
class TimeoutError < RuntimeError
end
class GracefulShutdown < RuntimeError
end
def run
@fiber = Fiber.current
@service.mount(route, self)
@mounted = true
# keep_alive_timer = spin_loop("#{@fiber.tag}-keep_alive", interval: 5) { keep_alive }
process_incoming_messages(false)
rescue GracefulShutdown
puts "Proxy got graceful shutdown, left: #{@requests.size} requests" if @requests.size > 0
move_on_after(15) { process_incoming_messages(true) }
ensure
# keep_alive_timer&.stop
unmount
end
def process_incoming_messages(shutdown = false)
return if shutdown && @requests.empty?
@conn.feed_loop(@msgpack_reader, :feed_each) do |msg|
recv_df_message(msg)
return if shutdown && @requests.empty?
end
rescue TimeoutError, IOError, SystemCallError
# ignore and just return in order to terminate the proxy
end
def unmount
return unless @mounted
@service.unmount(self)
@mounted = nil
end
def send_shutdown
send_df_message(Protocol.shutdown)
@fiber.raise GracefulShutdown.new
end
def keep_alive
now = Time.now
if now - @last_send >= Protocol::SEND_TIMEOUT
send_df_message(Protocol.ping)
end
# if now - @last_recv >= Protocol::RECV_TIMEOUT
# raise TimeoutError
# end
rescue TimeoutError, IOError
end
def route
case @req.headers['df-mount']
when /^\s*host\s*=\s*([^\s]+)/
{ host: Regexp.last_match(1) }
when /^\s*path\s*=\s*([^\s]+)/
{ path: Regexp.last_match(1) }
when /catch_all/
{ catch_all: true }
else
nil
end
end
def recv_df_message(message)
@last_recv = Time.now
# puts "<<< #{message.inspect}"
case message[Protocol::Attribute::KIND]
when Protocol::PING
return
when Protocol::UNMOUNT
return unmount
when Protocol::STATS_REQUEST
return handle_stats_request(message[Protocol::Attribute::ID])
end
handler = @requests[message[Protocol::Attribute::ID]]
if !handler
# puts "Unknown request id in #{message}"
return
end
handler << message
end
def send_df_message(message)
# puts ">>> #{message.inspect}" unless message[Protocol::Attribute::KIND] == Protocol::PING
@last_send = Time.now
@conn << message.to_msgpack
end
# HTTP / WebSocket
def register_request_fiber
id = (@last_request_id += 1)
@requests[id] = Fiber.current
id
end
def unregister_request_fiber(id)
@requests.delete(id)
end
def with_request
@current_request_count += 1
id = (@last_request_id += 1)
@requests[id] = Fiber.current
yield id
ensure
@current_request_count -= 1
@requests.delete(id)
end
def http_request(req)
t0 = Time.now
t1 = nil
with_request do |id|
msg = Protocol.http_request(id, req.headers, req.next_chunk(true), req.complete?)
send_df_message(msg)
while (message = receive)
kind = message[Protocol::Attribute::KIND]
unless t1
t1 = Time.now
if kind == Protocol::HTTP_RESPONSE
headers = message[Protocol::Attribute::HttpResponse::HEADERS]
status = (headers && headers[':status']) || 200
if status < Qeweney::Status::BAD_REQUEST
@service.record_latency_measurement(t1 - t0, req)
end
end
end
attributes = message[Protocol::Attribute::HttpRequest::HEADERS..-1]
return if http_request_message(id, req, kind, attributes)
end
end
rescue => e
p "Internal server error: #{e.inspect}"
puts e.backtrace.join("\n")
http_request_send_error_response(e)
end
def http_request_send_error_response(error)
response = format("Error: %s\n%s", error.inspect, error.backtrace.join("\n"))
req.respond(response, ':status' => Qeweney::Status::INTERNAL_SERVER_ERROR)
rescue IOError, SystemCallError
# ignore
end
# @return [Boolean] true if response is complete
def http_request_message(id, req, kind, message)
case kind
when Protocol::HTTP_UPGRADE
http_custom_upgrade(id, req, *message)
true
when Protocol::HTTP_GET_REQUEST_BODY
http_get_request_body(id, req, *message)
false
when Protocol::HTTP_RESPONSE
http_response(id, req, *message)
else
# invalid message
true
end
end
def send_transfer_count(key, rx, tx)
send_df_message(Protocol.transfer_count(key, rx, tx))
end
def handle_stats_request(id)
stats = @service.get_stats
send_df_message(Protocol.stats_response(id, stats))
end
HTTP_RESPONSE_UPGRADE_HEADERS = { ':status' => Qeweney::Status::SWITCHING_PROTOCOLS }
def http_custom_upgrade(id, req, headers)
# send upgrade response
upgrade_headers = headers ?
headers.merge(HTTP_RESPONSE_UPGRADE_HEADERS) :
HTTP_RESPONSE_UPGRADE_HEADERS
req.send_headers(upgrade_headers, true)
conn = req.adapter.conn
reader = spin("#{Fiber.current.tag}.#{id}") do
conn.recv_loop do |data|
send_df_message(Protocol.conn_data(id, data))
end
end
while (message = receive)
return if http_custom_upgrade_message(conn, message)
end
ensure
reader.stop
end
def http_custom_upgrade_message(conn, message)
case message[Protocol::Attribute::KIND]
when Protocol::CONN_DATA
conn << message[:Protocol::Attribute::ConnData::DATA]
false
when Protocol::CONN_CLOSE
true
else
# invalid message
true
end
end
def http_response(id, req, body, headers, complete, transfer_count_key)
if !req.headers_sent? && complete
req.respond(body, headers|| {})
if transfer_count_key
rx, tx = req.transfer_counts
send_transfer_count(transfer_count_key, rx, tx)
end
true
else
req.send_headers(headers) if headers && !req.headers_sent?
req.send_chunk(body, done: complete) if body or complete
if complete && transfer_count_key
rx, tx = req.transfer_counts
send_transfer_count(transfer_count_key, rx, tx)
end
complete
end
rescue IOError, SystemCallError
# ignore error
end
def http_get_request_body(id, req, limit)
case limit
when nil
body = req.read
else
limit = limit.to_i
body = nil
req.each_chunk do |chunk|
(body ||= +'') << chunk
break if body.bytesize >= limit
end
end
send_df_message(Protocol.http_request_body(id, body, req.complete?))
end
def http_upgrade(req, protocol)
if protocol == 'websocket'
handle_websocket_upgrade(req)
else
# other protocol upgrades should be handled by the agent, so we just run
# the request normally. The agent is expected to upgrade the connection
# using a http_upgrade message. From that moment on, two-way
# communication is handled using conn_data and conn_close messages.
http_request(req)
end
end
def handle_websocket_upgrade(req)
with_request do |id|
send_df_message(Protocol.ws_request(id, req.headers))
response = receive
case response[0]
when Protocol::WS_RESPONSE
headers = response[2] || {}
status = headers[':status'] || Qeweney::Status::SWITCHING_PROTOCOLS
if status != Qeweney::Status::SWITCHING_PROTOCOLS
req.respond(nil, headers)
return
end
ws = Tipi::Websocket.new(req.adapter.conn, req.headers)
run_websocket_connection(id, ws)
else
req.respond(nil, ':status' => Qeweney::Status::SERVICE_UNAVAILABLE)
end
end
rescue IOError, SystemCallError
# ignore
end
def run_websocket_connection(id, websocket)
reader = spin("#{Fiber.current}.#{id}-ws") do
websocket.recv_loop do |data|
send_df_message(Protocol.ws_data(id, data))
end
end
while (message = receive)
case message[Protocol::Attribute::KIND]
when Protocol::WS_DATA
websocket << message[Protocol::Attribute::WS::DATA]
when Protocol::WS_CLOSE
return
else
raise "Unexpected websocket message #{message.inspect}"
end
end
ensure
reader.stop
websocket.close
end
end
end
================================================
FILE: lib/tipi/digital_fabric/executive/index.html
================================================
<!doctype html>
<html lang="en">
<head>
<title>Digital Fabric Executive</title>
<style>
.hidden { display: none }
</style>
</head>
<body>
<h1>Digital Fabric Executive</h1>
<script>
function updateStats(update) {
for (let k in update.service) {
let v = update.service[k];
let e = document.querySelector('#' + k);
if (e) e.innerText = v;
}
for (let k in update.machine) {
let v = update.machine[k];
let e = document.querySelector('#' + k);
if (e) e.innerText = v;
}
}
function connect() {
console.log("connecting...");
window.eventSource = new EventSource("/stream/stats");
window.eventSource.onopen = function(e) {
console.log("connected");
document.querySelector('#status').innerText = 'connected';
document.querySelector('#stats').className = '';
return false;
}
window.eventSource.onmessage = function(e) {
console.log("message", e.data);
updateStats(JSON.parse(e.data));
}
window.eventSource.onerror = function(e) {
console.log("error", e);
document.querySelector('#status').innerText = 'disconnected';
document.querySelector('#stats').className = 'hidden';
window.eventSource.close();
window.eventSource = null;
setTimeout(connect, 5000);
}
};
window.onload = connect;
</script>
<h2 id="status"></h2>
<div id="stats" class="hidden">
<h2>Service</h2>
<p>Request rate: <span id="http_request_rate"></span></p>
<p>Error rate: <span id="error_rate"></span></p>
<p>Average Latency: <span id="average_latency"></span>s</p>
<p>Connected agents: <span id="agent_count"></span></p>
<p>Connected clients: <span id="connection_count"></span></p>
<p>Concurrent requests: <span id="concurrent_requests"></span></p>
<h2>Machine</h2>
<p>CPU utilization: <span id="cpu_utilization"></span>%</p>
<p>Free memory: <span id="mem_free"></span>MB</p>
<p>Load average: <span id="load_avg"></span></p>
</div>
</body>
</html>
================================================
FILE: lib/tipi/digital_fabric/executive.rb
================================================
# frozen_string_literal: true
require 'tipi/digital_fabric'
require 'json'
module DigitalFabric
# agent for managing DF service
class Executive
INDEX_HTML = IO.read(File.join(__dir__, 'executive/index.html'))
attr_reader :last_service_stats
def initialize(service, route = { path: '/executive' })
@service = service
route[:executive] = true
@service.mount(route, self)
@current_request_count = 0
# @updater = spin_loop(:executive_updater, interval: 10) { update_service_stats }
update_service_stats
end
def current_request_count
@current_request_count
end
def http_request(req)
@current_request_count += 1
case req.path
when '/'
req.respond(INDEX_HTML, 'Content-Type' => 'text/html')
when '/stats'
message = last_service_stats
req.respond(message.to_json, { 'Content-Type' => 'text.json' })
when '/stream/stats'
stream_stats(req)
when '/upload'
req.respond("body: #{req.read.inspect}")
else
req.respond('Invalid path', { ':status' => Qeweney::Status::NOT_FOUND })
end
rescue => e
puts "Error: #{e.inspect}"
ensure
@current_request_count -= 1
end
def stream_stats(req)
req.send_headers({ 'Content-Type' => 'text/event-stream' })
every(10) do
message = last_service_stats
req.send_chunk(format_sse_event(message.to_json))
end
rescue IOError, SystemCallError
# ignore
ensure
req.send_chunk("retry: 0\n\n", true) rescue nil
end
def format_sse_event(data)
"data: #{data}\n\n"
end
def update_service_stats
@last_service_stats = {
service: @service.stats,
machine: machine_stats
}
end
TOP_CPU_REGEXP = /%Cpu(.+)/.freeze
TOP_CPU_IDLE_REGEXP = /([\d\.]+) id/.freeze
TOP_MEM_REGEXP = /MiB Mem(.+)/.freeze
TOP_MEM_FREE_REGEXP = /([\d\.]+) free/.freeze
LOADAVG_REGEXP = /^([\d\.]+)/.freeze
def machine_stats
top = `top -bn1 | head -n4`
unless top =~ TOP_CPU_REGEXP && Regexp.last_match(1) =~ TOP_CPU_IDLE_REGEXP
p top =~ TOP_CPU_REGEXP
p Regexp.last_match(1)
p Regexp.last_match(1) =~ TOP_CPU_IDLE_REGEXP
raise 'Invalid output from top (cpu)'
end
cpu_utilization = 100 - Regexp.last_match(1).to_i
unless top =~ TOP_MEM_REGEXP && Regexp.last_match(1) =~ TOP_MEM_FREE_REGEXP
raise 'Invalid output from top (mem)'
end
mem_free = Regexp.last_match(1).to_f
stats = `cat /proc/loadavg`
raise 'Invalid output from /proc/loadavg' unless stats =~ LOADAVG_REGEXP
load_avg = Regexp.last_match(1).to_f
{
mem_free: mem_free,
cpu_utilization: cpu_utilization,
load_avg: load_avg
}
end
end
end
================================================
FILE: lib/tipi/digital_fabric/protocol.rb
================================================
# frozen_string_literal: true
module DigitalFabric
module Protocol
PING = 'ping'
SHUTDOWN = 'shutdown'
UNMOUNT = 'unmount'
HTTP_REQUEST = 'http_request'
HTTP_RESPONSE = 'http_response'
HTTP_UPGRADE = 'http_upgrade'
HTTP_GET_REQUEST_BODY = 'http_get_request_body'
HTTP_REQUEST_BODY = 'http_request_body'
CONN_DATA = 'conn_data'
CONN_CLOSE = 'conn_close'
WS_REQUEST = 'ws_request'
WS_RESPONSE = 'ws_response'
WS_DATA = 'ws_data'
WS_CLOSE = 'ws_close'
TRANSFER_COUNT = 'transfer_count'
STATS_REQUEST = 'stats_request'
STATS_RESPONSE = 'stats_response'
SEND_TIMEOUT = 15
RECV_TIMEOUT = SEND_TIMEOUT + 5
module Attribute
KIND = 0
ID = 1
module HttpRequest
HEADERS = 2
BODY_CHUNK = 3
COMPLETE = 4
end
module HttpResponse
BODY = 2
HEADERS = 3
COMPLETE = 4
TRANSFER_COUNT_KEY = 5
end
module HttpUpgrade
HEADERS = 2
end
module HttpGetRequestBody
LIMIT = 2
end
module HttpRequestBody
BODY = 2
COMPLETE = 3
end
module ConnectionData
DATA = 2
end
module WS
HEADERS = 2
DATA = 2
end
module TransferCount
KEY = 1
RX = 2
TX = 3
end
module Stats
STATS = 2
end
end
class << self
def ping
[ PING ]
end
def shutdown
[ SHUTDOWN ]
end
def unmount
[ UNMOUNT ]
end
DF_UPGRADE_RESPONSE = <<~HTTP.gsub("\n", "\r\n")
HTTP/1.1 101 Switching Protocols
Upgrade: df
Connection: Upgrade
HTTP
def df_upgrade_response
DF_UPGRADE_RESPONSE
end
def http_request(id, headers, buffered_chunk, complete)
[ HTTP_REQUEST, id, headers, buffered_chunk, complete ]
end
def http_response(id, body, headers, complete, transfer_count_key = nil)
[ HTTP_RESPONSE, id, body, headers, complete, transfer_count_key ]
end
def http_upgrade(id, headers)
[ HTTP_UPGRADE, id, headers ]
end
def http_get_request_body(id, limit = nil)
[ HTTP_GET_REQUEST_BODY, id, limit ]
end
def http_request_body(id, body, complete)
[ HTTP_REQUEST_BODY, id, body, complete ]
end
def connection_data(id, data)
[ CONN_DATA, id, data ]
end
def connection_close(id)
[ CONN_CLOSE, id ]
end
def ws_request(id, headers)
[ WS_REQUEST, id, headers ]
end
def ws_response(id, headers)
[ WS_RESPONSE, id, headers ]
end
def ws_data(id, data)
[ WS_DATA, id, data ]
end
def ws_close(id)
[ WS_CLOSE, id ]
end
def transfer_count(key, rx, tx)
[ TRANSFER_COUNT, key, rx, tx ]
end
def stats_request(id)
[ STATS_REQUEST, id ]
end
def stats_response(id, stats)
[ STATS_RESPONSE, id, stats ]
end
end
end
end
================================================
FILE: lib/tipi/digital_fabric/request_adapter.rb
================================================
# frozen_string_literal: true
require_relative './protocol'
module DigitalFabric
class RequestAdapter
def initialize(agent, msg)
@agent = agent
@id = msg[Protocol::Attribute::ID]
end
def protocol
'df'
end
def get_body_chunk(request)
@agent.get_http_request_body(@id, 1)
end
def respond(request, body, headers)
@agent.send_df_message(
Protocol.http_response(@id, body, headers, true)
)
end
def send_headers(request, headers, opts = {})
@agent.send_df_message(
Protocol.http_response(@id, nil, headers, false)
)
end
def send_chunk(request, body, done: )
@agent.send_df_message(
Protocol.http_response(@id, body, nil, done)
)
end
def finish(request)
@agent.send_df_message(
Protocol.http_response(@id, nil, nil, true)
)
end
end
end
================================================
FILE: lib/tipi/digital_fabric/service.rb
================================================
# frozen_string_literal: true
require_relative './protocol'
require_relative './agent_proxy'
require 'securerandom'
module DigitalFabric
class Service
attr_reader :token
attr_reader :timer
def initialize(token: )
@token = token
@agents = {}
@routes = {}
@counters = {
connections: 0,
http_requests: 0,
errors: 0
}
@connection_count = 0
@current_request_count = 0
@http_latency_accumulator = 0
@http_latency_counter = 0
@http_latency_max = 0
@last_counters = @counters.merge(stamp: Time.now.to_f - 1)
@fiber = Fiber.current
# @timer = Polyphony::Timer.new('service_timer', resolution: 5)
end
def calculate_stats
now = Time.now.to_f
elapsed = now - @last_counters[:stamp]
connections = @counters[:connections] - @last_counters[:connections]
http_requests = @counters[:http_requests] - @last_counters[:http_requests]
errors = @counters[:errors] - @last_counters[:errors]
@last_counters = @counters.merge(stamp: now)
average_latency = @http_latency_counter == 0 ? 0 :
@http_latency_accumulator / @http_latency_counter
@http_latency_accumulator = 0
@http_latency_counter = 0
max_latency = @http_latency_max
@http_latency_max = 0
cpu, rss = pid_cpu_and_rss(Process.pid)
backend_stats = Thread.backend.stats
op_rate = backend_stats[:op_count] / elapsed
switch_rate = backend_stats[:switch_count] / elapsed
poll_rate = backend_stats[:poll_count] / elapsed
object_space_stats = ObjectSpace.count_objects
{
service: {
agent_count: @agents.size,
connection_count: @connection_count,
connection_rate: connections / elapsed,
error_rate: errors / elapsed,
http_request_rate: http_requests / elapsed,
latency_avg: average_latency,
latency_max: max_latency,
pending_requests: @current_request_count,
},
backend: {
op_rate: op_rate,
pending_ops: backend_stats[:pending_ops],
poll_rate: poll_rate,
runqueue_size: backend_stats[:runqueue_size],
runqueue_high_watermark: backend_stats[:runqueue_max_length],
switch_rate: switch_rate,
},
process: {
cpu_usage: cpu,
rss: rss.to_f / 1024,
objects_total: object_space_stats[:TOTAL],
objects_free: object_space_stats[:FREE]
}
}
end
def pid_cpu_and_rss(pid)
s = `ps -p #{pid} -o %cpu,rss`
cpu, rss = s.lines[1].chomp.strip.split(' ')
[cpu.to_f, rss.to_i]
rescue Polyphony::BaseException
raise
rescue Exception
[nil, nil]
end
def get_stats
calculate_stats
end
def incr_connection_count
@connection_count += 1
end
def decr_connection_count
@connection_count -= 1
end
attr_reader :stats
def total_request_count
count = 0
@agents.keys.each do |agent|
if agent.respond_to?(:current_request_count)
count += agent.current_request_count
end
end
count
end
def record_latency_measurement(latency, req)
@http_latency_accumulator += latency
@http_latency_counter += 1
@http_latency_max = latency if latency > @http_latency_max
return if latency < 1.0
puts format('slow request (%.1f): %p', latency, req.headers)
end
def http_request(req, allow_df_upgrade = false)
@current_request_count += 1
@counters[:http_requests] += 1
@counters[:connections] += 1 if req.headers[':first']
return upgrade_request(req, allow_df_upgrade) if req.upgrade_protocol
inject_request_headers(req)
agent = find_agent(req)
unless agent
@counters[:errors] += 1
return req.respond(nil, ':status' => Qeweney::Status::SERVICE_UNAVAILABLE)
end
agent.http_request(req)
rescue IOError, SystemCallError, HTTP2::Error::StreamClosed
@counters[:errors] += 1
rescue => e
@counters[:errors] += 1
puts '*' * 40
p req
p e
puts e.backtrace.join("\n")
req.respond(e.inspect, ':status' => Qeweney::Status::INTERNAL_SERVER_ERROR)
ensure
@current_request_count -= 1
req.adapter.conn.close if @shutdown
end
def inject_request_headers(req)
req.headers['x-request-id'] = SecureRandom.uuid
conn = req.adapter.conn
req.headers['x-forwarded-for'] = conn.peeraddr(false)[2]
req.headers['x-forwarded-proto'] ||= conn.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
end
def upgrade_request(req, allow_df_upgrade)
case (protocol = req.upgrade_protocol)
when 'df'
if allow_df_upgrade
df_upgrade(req)
else
req.respond(nil, ':status' => Qeweney::Status::SERVICE_UNAVAILABLE)
end
else
agent = find_agent(req)
unless agent
@counters[:errors] += 1
return req.respond(nil, ':status' => Qeweney::Status::SERVICE_UNAVAILABLE)
end
agent.http_upgrade(req, protocol)
end
end
def df_upgrade(req)
# we don't want to count connected agents
@current_request_count -= 1
if req.headers['df-token'] != @token
return req.respond(nil, ':status' => Qeweney::Status::FORBIDDEN)
end
req.adapter.conn << Protocol.df_upgrade_response
AgentProxy.new(self, req)
ensure
@current_request_count += 1
end
def mount(route, agent)
if route[:path]
route[:path_regexp] = path_regexp(route[:path])
end
@executive = agent if route[:executive]
@agents[agent] = route
@routing_changed = true
end
def unmount(agent)
route = @agents[agent]
return unless route
@executive = nil if route[:executive]
@agents.delete(agent)
@routing_changed = true
end
INVALID_HOST = 'INVALID_HOST'
def find_agent(req)
compile_agent_routes if @routing_changed
host = req.headers[':authority'] || req.headers['host'] || INVALID_HOST
path = req.headers[':path']
route = @route_keys.find do |route|
(host == route[:host]) || (path =~ route[:path_regexp])
end
return @routes[route] if route
nil
end
def compile_agent_routes
@routing_changed = false
@routes.clear
@agents.keys.reverse.each do |agent|
route = @agents[agent]
@routes[route] ||= agent
end
@route_keys = @routes.keys
end
def path_regexp(path)
/^#{path}/
end
def graceful_shutdown
@shutdown = true
@agents.keys.each do |agent|
if agent.respond_to?(:send_shutdown)
agent.send_shutdown
else
@agents.delete(agent)
end
end
move_on_after(60) do
while !@agents.empty?
sleep 0.25
end
end
end
end
end
================================================
FILE: lib/tipi/digital_fabric.rb
================================================
module DigitalFabric
end
::DF = DigitalFabric
require_relative 'digital_fabric/service'
require_relative 'digital_fabric/agent_proxy'
================================================
FILE: lib/tipi/handler.rb
================================================
# frozen_string_literal: true
require_relative './rack_adapter'
require_relative './http1_adapter'
require_relative './http2_adapter'
module Tipi
class DefaultHandler
def initialize(config)
@config = config
app_path = ARGV.first || './config.ru'
@app = Tipi::RackAdapter.load(app_path)
end
def call(socket)
socket.no_delay if socket.respond_to?(:no_delay)
adapter = protocol_adapter(socket, {})
adapter.each(&@app)
ensure
socket.close
end
ALPN_PROTOCOLS = %w[h2 http/1.1].freeze
H2_PROTOCOL = 'h2'
def protocol_adapter(socket, opts)
use_http2 = socket.respond_to?(:alpn_protocol) &&
socket.alpn_protocol == H2_PROTOCOL
klass = use_http2 ? HTTP2Adapter : HTTP1Adapter
klass.new(socket, opts)
end
end
end
================================================
FILE: lib/tipi/http1_adapter.rb
================================================
# frozen_string_literal: true
require 'h1p'
require 'qeweney/request'
require_relative './http2_adapter'
module Tipi
# HTTP1 protocol implementation
class HTTP1Adapter
attr_reader :conn
# Initializes a protocol adapter instance
def initialize(conn, opts)
@conn = conn
@opts = opts
@first = true
@parser = H1P::Parser.new(@conn, :server)
end
def each(&block)
while true
headers = @parser.parse_headers
break unless headers
# handle_request returns true if connection is not persistent or was
# upgraded
break if handle_request(headers, &block)
end
rescue SystemCallError, IOError, H1P::Error
# connection or parser error, ignore
ensure
finalize_client_loop
end
def handle_request(headers, &block)
scheme = (proto = headers['x-forwarded-proto']) ?
proto.downcase : scheme_from_connection
headers[':scheme'] = scheme
@protocol = headers[':protocol']
if @first
headers[':first'] = true
@first = nil
end
return true if upgrade_connection(headers, &block)
request = Qeweney::Request.new(headers, self)
if !@parser.complete?
request.buffer_body_chunk(@parser.read_body_chunk(true))
end
block.call(request)
return !persistent_connection?(headers)
end
def persistent_connection?(headers)
if headers[':protocol'] == 'http/1.1'
return headers['connection'] != 'close'
else
connection = headers['connection']
return connection && connection != 'close'
end
end
def finalize_client_loop
@parser = nil
@splicing_pipe = nil
@conn.shutdown if @conn.respond_to?(:shutdown) rescue nil
@conn.close
end
# Reads a body chunk for the current request. Transfers control to the parse
# loop, and resumes once the parse_loop has fired the on_body callback
def get_body_chunk(request, buffered_only = false)
@parser.read_body_chunk(buffered_only)
end
def get_body(request)
@parser.read_body
end
def complete?(request)
@parser.complete?
end
def protocol
@protocol
end
# Upgrades the connection to a different protocol, if the 'Upgrade' header is
# given. By default the only supported upgrade protocol is HTTP2. Additional
# protocols, notably WebSocket, can be specified by passing a hash to the
# :upgrade option when starting a server:
#
# def ws_handler(conn)
# conn << 'hi'
# msg = conn.recv
# conn << "You said #{msg}"
# conn << 'bye'
# conn.close
# end
#
# opts = {
# upgrade: {
# websocket: Tipi::Websocket.handler(&method(:ws_handler))
# }
# }
# Tipi.serve('0.0.0.0', 1234, opts) { |req| ... }
#
# @param headers [Hash] request headers
# @return [boolean] truthy if the connection has been upgraded
def upgrade_connection(headers, &block)
upgrade_protocol = headers['upgrade']
return nil unless upgrade_protocol
upgrade_protocol = upgrade_protocol.downcase.to_sym
upgrade_handler = @opts[:upgrade] && @opts[:upgrade][upgrade_protocol]
return upgrade_with_handler(upgrade_handler, headers) if upgrade_handler
return upgrade_to_http2(headers, &block) if upgrade_protocol == :h2c
nil
end
def upgrade_with_handler(handler, headers)
@parser = nil
handler.(self, headers)
true
end
def upgrade_to_http2(headers, &block)
headers = http2_upgraded_headers(headers)
body = @parser.read_body
HTTP2Adapter.upgrade_each(@conn, @opts, headers, body, &block)
true
end
# Returns headers for HTTP2 upgrade
# @param headers [Hash] request headers
# @return [Hash] headers for HTTP2 upgrade
def http2_upgraded_headers(headers)
headers.merge(
':scheme' => 'http',
':authority' => headers['host']
)
end
def websocket_connection(request)
Tipi::Websocket.new(@conn, request.headers)
end
def scheme_from_connection
@conn.is_a?(OpenSSL::SSL::SSLSocket) ? 'https' : 'http'
end
# response API
CRLF = "\r\n"
# Sends response including headers and body. Waits for the request to complete
# if not yet completed. The body is sent using chunked transfer encoding.
# @param request [Qeweney::Request] HTTP request
# @param body [String] response body
# @param headers
def respond(request, body, headers)
written = H1P.send_response(@conn, headers, body)
request.tx_incr(written)
end
CHUNK_LENGTH_PROC = ->(len) { "#{len.to_s(16)}\r\n" }
def respond_from_io(request, io, headers, chunk_size = 2**14)
formatted_headers = format_headers(headers, true, true)
request.tx_incr(formatted_headers.bytesize)
# assume chunked encoding
Thread.current.backend.splice_chunks(
io,
@conn,
formatted_headers,
"0\r\n\r\n",
CHUNK_LENGTH_PROC,
"\r\n",
chunk_size
)
end
# Sends response headers. If empty_response is truthy, the response status
# code will default to 204, otherwise to 200.
# @param request [Qeweney::Request] HTTP request
# @param headers [Hash] response headers
# @param empty_response [boolean] whether a response body will be sent
# @param chunked [boolean] whether to use chunked transfer encoding
# @return [void]
def send_headers(request, headers, empty_response: false, chunked: true)
formatted_headers = format_headers(headers, !empty_response, http1_1?(request) && chunked)
request.tx_incr(formatted_headers.bytesize)
@conn.write(formatted_headers)
end
def http1_1?(request)
request.headers[':protocol'] == 'http/1.1'
end
# Sends a response body chunk. If no headers were sent, default headers are
# sent using #send_headers. if the done option is true(thy), an empty chunk
# will be sent to signal response completion to the client.
# @param request [Qeweney::Request] HTTP request
# @param chunk [String] response body chunk
# @param done [boolean] whether the response is completed
# @return [void]
def send_chunk(request, chunk, done: false)
if done
data = chunk ?
"#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n0\r\n\r\n" :
"0\r\n\r\n"
elsif chunk
data = "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
else
return
end
request.tx_incr(data.bytesize)
@conn.write(data)
end
def send_chunk_from_io(request, io, r, w, chunk_size)
len = w.splice(io, chunk_size)
if len > 0
Thread.current.backend.chain(
[:write, @conn, "#{len.to_s(16)}\r\n"],
[:splice, r, @conn, len],
[:write, @conn, "\r\n"]
)
else
@conn.write("0\r\n\r\n")
end
len
end
# Finishes the response to the current request. If no headers were sent,
# default headers are sent using #send_headers.
# @return [void]
def finish(request)
request.tx_incr(5)
@conn << "0\r\n\r\n"
end
def close
@conn.shutdown if @conn.respond_to?(:shutdown) rescue nil
@conn.close
end
private
INTERNAL_HEADER_REGEXP = /^:/.freeze
# Formats response headers into an array. If empty_response is true(thy),
# the response status code will default to 204, otherwise to 200.
# @param headers [Hash] response headers
# @param body [boolean] whether a response body will be sent
# @param chunked [boolean] whether to use chunked transfer encoding
# @return [String] formatted response headers
def format_headers(headers, body, chunked)
status = headers[':status']
status ||= (body ? Qeweney::Status::OK : Qeweney::Status::NO_CONTENT)
lines = format_status_line(body, status, chunked)
headers.each do |k, v|
next if k =~ INTERNAL_HEADER_REGEXP
collect_header_lines(lines, k, v)
end
lines << CRLF
lines
end
def format_status_line(body, status, chunked)
if !body
empty_status_line(status)
else
with_body_status_line(status, body, chunked)
end
end
def empty_status_line(status)
if status == 204
+"HTTP/1.1 #{status}\r\n"
else
+"HTTP/1.1 #{status}\r\nContent-Length: 0\r\n"
end
end
def with_body_status_line(status, body, chunked)
if chunked
+"HTTP/1.1 #{status}\r\nTransfer-Encoding: chunked\r\n"
else
+"HTTP/1.1 #{status}\r\nContent-Length: #{body.is_a?(String) ? body.bytesize : body.to_i}\r\n"
end
end
def collect_header_lines(lines, key, value)
if value.is_a?(Array)
value.inject(lines) { |_, item| lines << "#{key}: #{item}\r\n" }
else
lines << "#{key}: #{value}\r\n"
end
end
end
end
================================================
FILE: lib/tipi/http2_adapter.rb
================================================
# frozen_string_literal: true
require 'http/2'
require_relative './http2_stream'
# patch to fix bug in HTTP2::Stream
class HTTP2::Stream
def end_stream?(frame)
case frame[:type]
when :data, :headers, :continuation
frame[:flags]&.include?(:end_stream)
else false
end
end
end
module Tipi
# HTTP2 server adapter
class HTTP2Adapter
def self.upgrade_each(socket, opts, headers, body, &block)
adapter = new(socket, opts, headers, body)
adapter.each(&block)
end
def initialize(conn, opts, upgrade_headers = nil, upgrade_body = nil)
@conn = conn
@opts = opts
@upgrade_headers = upgrade_headers
@upgrade_body = upgrade_body
@first = true
@rx = (upgrade_headers && upgrade_headers[':rx']) || 0
@tx = (upgrade_headers && upgrade_headers[':tx']) || 0
@interface = ::HTTP2::Server.new
@connection_fiber = Fiber.current
@interface.on(:frame, &method(:send_frame))
@streams = {}
end
def send_frame(data)
if @transfer_count_request
@transfer_count_request.tx_incr(data.bytesize)
end
@conn << data
rescue Polyphony::BaseException
raise
rescue Exception => e
@connection_fiber.transfer e
end
UPGRADE_MESSAGE = <<~HTTP.gsub("\n", "\r\n")
HTTP/1.1 101 Switching Protocols
Connection: Upgrade
Upgrade: h2c
HTTP
def upgrade
@conn << UPGRADE_MESSAGE
@tx += UPGRADE_MESSAGE.bytesize
settings = @upgrade_headers['http2-settings']
@interface.upgrade(settings, @upgrade_headers, @upgrade_body || '')
ensure
@upgrade_headers = nil
end
# Iterates over incoming requests
def each(&block)
@interface.on(:stream) { |stream| start_stream(stream, &block) }
upgrade if @upgrade_headers
@conn.recv_loop do |data|
@rx += data.bytesize
@interface << data
end
rescue SystemCallError, IOError, HTTP2::Error::Error
# ignore
ensure
finalize_client_loop
end
def get_rx_count
count = @rx
@rx = 0
count
end
def get_tx_count
count = @tx
@tx = 0
count
end
def start_stream(stream, &block)
stream = HTTP2StreamHandler.new(self, stream, @conn, @first, &block)
@first = nil if @first
@streams[stream] = true
end
def finalize_client_loop
@interface = nil
@streams.each_key(&:stop)
@conn.shutdown if @conn.respond_to?(:shutdown) rescue nil
@conn.close
end
def close
@conn.shutdown if @conn.respond_to?(:shutdown) rescue nil
@conn.close
end
def set_request_for_transfer_count(request)
@transfer_count_request = request
end
def unset_request_for_transfer_count(request)
return unless @transfer_count_request == request
@transfer_count_request = nil
end
end
end
================================================
FILE: lib/tipi/http2_stream.rb
================================================
# frozen_string_literal: true
require 'http/2'
require 'qeweney/request'
module Tipi
# Manages an HTTP 2 stream
class HTTP2StreamHandler
attr_reader :conn
def initialize(adapter, stream, conn, first, &block)
@adapter = adapter
@stream = stream
@conn = conn
@first = first
@connection_fiber = Fiber.current
@stream_fiber = spin { run(&block) }
# Stream callbacks occur on the connection fiber (see HTTP2Adapter#each).
# The request handler is run on a separate fiber for each stream, allowing
# concurrent handling of incoming requests on the same HTTP/2 connection.
#
# The different stream adapter APIs suspend the stream fiber, waiting for
# stream callbacks to be called. The callbacks, in turn, transfer control to
# the stream fiber, effectively causing the return of the adapter API calls.
#
# Note: the request handler is run once headers are received. Reading the
# request body, if present, is at the discretion of the request handler.
# This mirrors the behaviour of the HTTP/1 adapter.
stream.on(:headers, &method(:on_headers))
stream.on(:data, &method(:on_data))
stream.on(:half_close, &method(:on_half_close))
end
def run(&block)
request = receive
error = nil
block.(request)
@connection_fiber.schedule
rescue Polyphony::BaseException
raise
rescue Exception => e
error = e
ensure
@connection_fiber.schedule error
end
def on_headers(headers)
@request = Qeweney::Request.new(headers.to_h, self)
@request.rx_incr(@adapter.get_rx_count)
@request.tx_incr(@adapter.get_tx_count)
if @first
@request.headers[':first'] = true
@first = false
end
@stream_fiber << @request
end
def on_data(data)
data = data.to_s # chunks might be wrapped in a HTTP2::Buffer
(@buffered_chunks ||= []) << data
@get_body_chunk_fiber&.schedule
end
def on_half_close
@get_body_chunk_fiber&.schedule
@complete = true
end
def protocol
'h2'
end
def with_transfer_count(request)
@adapter.set_request_for_transfer_count(request)
yield
ensure
@adapter.unset_request_for_transfer_count(request)
end
def get_body_chunk(request, buffered_only = false)
@buffered_chunks ||= []
return @buffered_chunks.shift unless @buffered_chunks.empty?
return nil if @complete
begin
@get_body_chunk_fiber = Fiber.current
suspend
ensure
@get_body_chunk_fiber = nil
end
@buffered_chunks.shift
end
def get_body(request)
@buffered_chunks ||= []
return @buffered_chunks.join if @complete
while !@complete
begin
@get_body_chunk_fiber = Fiber.current
suspend
ensure
@get_body_chunk_fiber = nil
end
end
@buffered_chunks.join
end
def complete?(request)
@complete
end
# response API
def respond(request, body, headers)
headers = normalize_status_header(headers)
with_transfer_count(request) do
@stream.headers(transform_headers(headers))
@headers_sent = true
@stream.data(body || '')
end
rescue HTTP2::Error::StreamClosed
# ignore
end
def respond_from_io(request, io, headers, chunk_size = 2**16)
headers = normalize_status_header(headers)
with_transfer_count(request) do
@stream.headers(transform_headers(headers))
@headers_sent = true
while (chunk = io.read(chunk_size))
@stream.data(chunk)
end
end
rescue HTTP2::Error::StreamClosed
# ignore
end
def transform_headers(headers)
headers.each_with_object([]) do |(k, v), a|
if v.is_a?(Array)
v.each { |vv| a << [k, vv.to_s] }
else
a << [k, v.to_s]
end
end
end
def send_headers(request, headers, empty_response: false)
return if @headers_sent
status = empty_response ? Qeweney::Status::NO_CONTENT : Qeweney::Status::OK
headers = normalize_status_header(headers, status)
with_transfer_count(request) do
@stream.headers(transform_headers(headers), end_stream: false)
end
@headers_sent = true
rescue HTTP2::Error::StreamClosed
# ignore
end
def send_chunk(request, chunk, done: false)
send_headers({}, false) unless @headers_sent
if chunk
with_transfer_count(request) do
@stream.data(chunk, end_stream: done)
end
elsif done
@stream.close
end
rescue HTTP2::Error::StreamClosed
# ignore
end
def finish(request)
if @headers_sent
@stream.close
else
headers[':status'] ||= Qeweney::Status::NO_CONTENT
with_transfer_count(request) do
@stream.headers(transform_headers(headers), end_stream: true)
end
end
rescue HTTP2::Error::StreamClosed
# ignore
end
def stop
return if @complete
@stream.close
@stream_fiber.schedule(Polyphony::MoveOn.new)
end
private
def normalize_status_header(headers, default_status = Qeweney::Status::OK)
if !headers[':status']
headers.merge(':status' => default_status.to_s)
elsif !headers[':status'].is_a?(String)
headers.merge(headers[':status'].to_s)
else
headers
end
end
end
end
================================================
FILE: lib/tipi/rack_adapter.rb
================================================
# frozen_string_literal: true
require 'rack'
module Tipi
module RackAdapter
class << self
def run(app)
->(req) { respond(req, app.(env(req))) }
end
def load(path)
src = IO.read(path)
instance_eval(src, path, 1)
end
def env(request)
Qeweney.rack_env_from_request(request)
end
def respond(request, (status_code, headers, body))
headers[':status'] = status_code.to_s
content =
if body.respond_to?(:to_path)
File.open(body.to_path, 'rb') { |f| f.read }
else
body.first
end
request.respond(content, headers)
end
end
end
end
================================================
FILE: lib/tipi/response_extensions.rb
================================================
# frozen_string_literal: true
require 'qeweney/request'
module Tipi
module ResponseExtensions
SPLICE_CHUNKS_SIZE_THRESHOLD = 2**20
def serve_io(io, opts)
if !opts[:stat] || opts[:stat].size >= SPLICE_CHUNKS_SIZE_THRESHOLD
@adapter.respond_from_io(self, io, opts[:headers], opts[:chunk_size] || 2**14)
else
respond(io.read, opts[:headers] || {})
end
end
end
end
================================================
FILE: lib/tipi/supervisor.rb
================================================
# frozen_string_literal: true
require 'polyphony'
require 'json'
module Tipi
module Supervisor
class << self
def run(opts)
puts "Start supervisor pid: #{Process.pid}"
@opts = opts
@controller_watcher = start_controller_watcher
supervise_loop
end
def start_controller_watcher
spin do
cmd = controller_cmd
puts "Starting controller..."
pid = Kernel.spawn(*cmd)
@controller_pid = pid
puts "Controller pid: #{pid}"
_pid, status = Polyphony.backend_waitpid(pid)
puts "Controller has terminated with status: #{status.inspect}"
terminated = true
ensure
if pid && !terminated
puts "Terminate controller #{pid.inspect}"
Polyphony::Process.kill_process(pid)
end
Fiber.current.parent << pid
end
end
def controller_cmd
[
'ruby',
File.join(__dir__, 'controller.rb'),
@opts.to_json
]
end
def supervise_loop
this_fiber = Fiber.current
trap('SIGUSR2') { this_fiber << :replace_controller }
loop do
case (msg = receive)
when :replace_controller
replace_controller
when Integer
pid = msg
if pid == @controller_pid
puts 'Detected dead controller. Restarting...'
exit!
@controller_watcher.restart
end
else
raise "Invalid message received: #{msg.inspect}"
end
end
end
def replace_controller
puts "Replacing controller"
old_watcher = @controller_watcher
@controller_watcher = start_controller_watcher
# TODO: we'll want to get some kind of signal from the new controller once it's ready
sleep 1
old_watcher.terminate(graceful: true)
end
end
end
end
================================================
FILE: lib/tipi/version.rb
================================================
# frozen_string_literal: true
module Tipi
VERSION = '0.56'
end
================================================
FILE: lib/tipi/websocket.rb
================================================
# frozen_string_literal: true
require 'digest/sha1'
require 'websocket'
module Tipi
# Websocket connection
class Websocket
def self.handler(&block)
proc do |adapter, headers|
req = Qeweney::Request.new(headers, adapter)
websocket = req.upgrade_to_websocket
block.(websocket)
end
end
def initialize(conn, headers)
@conn = conn
@headers = headers
@version = headers['sec-websocket-version'].to_i
@reader = ::WebSocket::Frame::Incoming::Server.new(version: @version)
end
def recv
if (msg = @reader.next)
return msg.to_s
end
@conn.recv_loop do |data|
@reader << data
if (msg = @reader.next)
return msg.to_s
end
end
nil
end
def recv_loop
if (msg = @reader.next)
yield msg.to_s
end
@conn.recv_loop do |data|
@reader << data
while (msg = @reader.next)
yield msg.to_s
end
end
end
OutgoingFrame = ::WebSocket::Frame::Outgoing::Server
def send(data)
frame = OutgoingFrame.new(
version: @version, data: data, type: :text
)
@conn << frame.to_s
end
alias_method :<<, :send
def close
@conn.close
end
end
end
================================================
FILE: lib/tipi.rb
================================================
# frozen_string_literal: true
require 'polyphony'
require_relative './tipi/http1_adapter'
require_relative './tipi/http2_adapter'
require_relative './tipi/configuration'
require_relative './tipi/response_extensions'
require_relative './tipi/acme'
require 'qeweney/request'
class Qeweney::Request
include Tipi::ResponseExtensions
end
module Tipi
ALPN_PROTOCOLS = %w[h2 http/1.1].freeze
H2_PROTOCOL = 'h2'
class << self
def serve(host, port, opts = {}, &handler)
opts[:alpn_protocols] = ALPN_PROTOCOLS
server = Polyphony::Net.tcp_listen(host, port, opts)
accept_loop(server, opts, &handler)
ensure
server&.close
end
def listen(host, port, opts = {})
opts[:alpn_protocols] = ALPN_PROTOCOLS
Polyphony::Net.tcp_listen(host, port, opts).tap do |socket|
socket.define_singleton_method(:each) do |&block|
::Tipi.accept_loop(socket, opts, &block)
end
end
end
def accept_loop(server, opts, &handler)
server.accept_loop do |client|
spin { client_loop(client, opts, &handler) }
rescue OpenSSL::SSL::SSLError
# disregard
end
end
def client_loop(client, opts, &handler)
client.no_delay if client.respond_to?(:no_delay)
adapter = protocol_adapter(client, opts)
adapter.each(&handler)
rescue SystemCallError
# disregard
ensure
client.close rescue nil
end
def protocol_adapter(socket, opts)
use_http2 = socket.respond_to?(:alpn_protocol) &&
socket.alpn_protocol == H2_PROTOCOL
klass = use_http2 ? HTTP2Adapter : HTTP1Adapter
klass.new(socket, opts)
end
def route(&block)
proc { |req| req.route(&block) }
end
end
end
================================================
FILE: test/coverage.rb
================================================
# frozen_string_literal: true
require 'coverage'
require 'simplecov'
class << SimpleCov::LinesClassifier
alias_method :orig_whitespace_line?, :whitespace_line?
def whitespace_line?(line)
line.strip =~ /^(begin|end|ensure|else|\})|(\s*rescue\s.+)$/ || orig_whitespace_line?(line)
end
end
module Coverage
EXCLUDE = %w{coverage eg helper run
}.map { |n| File.expand_path("test/#{n}.rb") }
LIB_FILES = Dir["#{File.join(FileUtils.pwd, 'lib')}/polyphony/**/*.rb"]
class << self
def relevant_lines_for_filename(filename)
@classifier ||= SimpleCov::LinesClassifier.new
@classifier.classify(IO.read(filename).lines)
end
def start
@result = {}
trace = TracePoint.new(:line) do |tp|
next if tp.path =~ /\(/
absolute = File.expand_path(tp.path)
next unless LIB_FILES.include?(absolute)# =~ /^#{LIB_DIR}/
@result[absolute] ||= relevant_lines_for_filename(absolute)
@result[absolute][tp.lineno - 1] = 1
end
trace.enable
end
def result
@result
end
end
end
SimpleCov.start
================================================
FILE: test/eg.rb
================================================
# frozen_string_literal: true
module Kernel
RE_CONST = /^[A-Z]/.freeze
RE_ATTR = /^@(.+)$/.freeze
def eg(hash)
Module.new.tap do |m|
s = m.singleton_class
hash.each do |k, v|
case k
when RE_CONST
m.const_set(k, v)
when RE_ATTR
m.instance_variable_set(k, v)
else
block = if v.respond_to?(:to_proc)
proc { |*args, &block| instance_exec { v.(*args, &block) } }
else
proc { v }
end
s.define_method(k, &block)
end
end
end
end
end
================================================
FILE: test/helper.rb
================================================
# frozen_string_literal: true
require 'bundler/setup'
require 'fileutils'
require_relative './eg'
require_relative './coverage' if ENV['COVERAGE']
require 'minitest/autorun'
require 'polyphony'
::Exception.__disable_sanitized_backtrace__ = true
class Minitest::Test
def setup
# trace "* setup #{self.name}"
Fiber.current.setup_main_fiber
Fiber.current.instance_variable_set(:@auto_watcher, nil)
Thread.current.backend.finalize
Thread.current.backend = Polyphony::Backend.new
sleep 0
end
def teardown
# trace "* teardown #{self.name}"
Fiber.current.shutdown_all_children
if Fiber.current.children.size > 0
puts "Children left after #{self.name}: #{Fiber.current.children.inspect}"
exit!
end
Fiber.current.instance_variable_set(:@auto_watcher, nil)
rescue => e
puts e
puts e.backtrace.join("\n")
exit!
end
end
module Kernel
def capture_exception
yield
rescue Exception => e
e
end
def trace(*args)
STDOUT.orig_write(format_trace(args))
end
def format_trace(args)
if args.first.is_a?(String)
if args.size > 1
format("%s: %p\n", args.shift, args)
else
format("%s\n", args.first)
end
else
format("%p\n", args.size == 1 ? args.first : args)
end
end
end
class IO
# Creates two mockup sockets for simulating server-client communication
def self.server_client_mockup
server_in, client_out = IO.pipe
client_in, server_out = IO.pipe
server_connection = mockup_connection(server_in, server_out, client_out)
client_connection = mockup_connection(client_in, client_out, server_out)
[server_connection, client_connection]
end
def self.mockup_connection(input, output, output2)
eg(
__read_method__: -> { :readpartial },
read: ->(*args) { input.read(*args) },
read_loop: ->(*args, &block) { input.read_loop(*args, &block) },
recv_loop: ->(*args, &block) { input.read_loop(*args, &block) },
readpartial: ->(*args) { input.readpartial(*args) },
recv: ->(*args) { input.readpartial(*args) },
'<<': ->(*args) { output.write(*args) },
write: ->(*args) { output.write(*args) },
close: -> { output.close },
eof?: -> { output2.closed? }
)
end
end
module Minitest::Assertions
def assert_in_range exp_range, act
msg = message(msg) { "Expected #{mu_pp(act)} to be in range #{mu_pp(exp_range)}" }
assert exp_range.include?(act), msg
end
end
================================================
FILE: test/run.rb
================================================
# frozen_string_literal: true
Dir.glob("#{__dir__}/test_*.rb").each do |path|
require(path)
end
================================================
FILE: test/test_http_server.rb
================================================
# frozen_string_literal: true
require_relative 'helper'
require 'tipi'
class String
def crlf_lines
gsub "\n", "\r\n"
end
end
class HTTP1ServerTest < Minitest::Test
def teardown
@server&.interrupt if @server&.alive?
sleep 0.01
super
end
def spin_server(opts = {}, &handler)
server_connection, client_connection = IO.server_client_mockup
coproc = spin do
Tipi.client_loop(server_connection, opts, &handler)
end
[coproc, client_connection, server_connection]
end
def test_that_server_uses_content_length_in_http_1_0
@server, connection = spin_server do |req|
req.respond('Hello, world!', {})
end
# using HTTP 1.0, server should close connection after responding
connection << "GET / HTTP/1.0\r\n\r\n"
response = connection.readpartial(8192)
expected = <<~HTTP.chomp.crlf_lines.chomp
HTTP/1.1 200 OK
Content-Length: 13
Hello, world!
HTTP
assert_equal(expected, response)
end
def test_that_server_uses_chunked_encoding_in_http_1_1
@server, connection = spin_server do |req|
req.respond('Hello, world!')
end
# using HTTP 1.0, server should close connection after responding
connection << "GET / HTTP/1.1\r\n\r\n"
response = connection.readpartial(8192)
expected = <<~HTTP.crlf_lines.chomp
HTTP/1.1 200 OK
Content-Length: 13
Hello, world!
HTTP
assert_equal(expected, response)
end
def test_that_server_maintains_connection_when_using_keep_alives
@server, connection = spin_server do |req|
req.respond('Hi', {})
end
connection << "GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n"
response = connection.readpartial(8192)
sleep 0.01
assert !connection.eof?
assert_equal("HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nHi", response)
connection << "GET / HTTP/1.1\r\n\r\n"
response = connection.readpartial(8192)
sleep 0.01
assert !connection.eof?
expected = <<~HTTP.crlf_lines.chomp
HTTP/1.1 200 OK
Content-Length: 2
Hi
HTTP
assert_equal(expected, response)
connection << "GET / HTTP/1.0\r\n\r\n"
response = connection.readpartial(8192)
sleep 0.01
assert connection.eof?
assert_equal("HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nHi", response)
end
def test_pipelining_client
@server, connection = spin_server do |req|
if req.headers['foo'] == 'bar'
req.respond('Hello, foobar!', {})
else
req.respond('Hello, world!', {})
end
end
connection << "GET / HTTP/1.1\r\n\r\nGET / HTTP/1.1\r\nFoo: bar\r\n\r\n"
sleep 0.01
response = connection.readpartial(8192)
expected = <<~HTTP.crlf_lines.chomp
HTTP/1.1 200 OK
Content-Length: 13
Hello, world!HTTP/1.1 200 OK
Content-Length: 14
Hello, foobar!
HTTP
assert_equal(expected, response)
end
def test_body_chunks
chunks = []
request = nil
@server, connection = spin_server do |req|
request = req
req.send_headers
req.each_chunk do |c|
chunks << c
req << c.upcase
end
req.finish
end
connection << <<~HTTP.crlf_lines
POST / HTTP/1.1
Transfer-Encoding: chunked
6
foobar
HTTP
sleep 0.01
assert request
assert_equal %w[foobar], chunks
assert !request.complete?
connection << "6\r\nbazbud\r\n"
sleep 0.01
assert_equal %w[foobar bazbud], chunks
assert !request.complete?
connection << "0\r\n\r\n"
sleep 0.01
assert_equal %w[foobar bazbud], chunks
assert request.complete?
sleep 0.01
response = connection.readpartial(8192)
expected = <<~HTTP.crlf_lines
HTTP/1.1 200
Transfer-Encoding: chunked
6
FOOBAR
6
BAZBUD
0
HTTP
assert_equal(expected, response)
end
def test_upgrade
done = nil
opts = {
upgrade: {
echo: lambda do |adapter, _headers|
conn = adapter.conn
conn << <<~HTTP.crlf_lines
HTTP/1.1 101 Switching Protocols
Upgrade: echo
Connection: Upgrade
HTTP
conn.read_loop { |data| conn << data }
done = true
end
}
}
@server, connection = spin_server(opts) do |req|
req.respond('Hi')
end
connection << "GET / HTTP/1.1\r\n\r\n"
response = connection.readpartial(8192)
sleep 0.01
assert !connection.eof?
expected = <<~HTTP.crlf_lines.chomp
HTTP/1.1 200 OK
Content-Length: 2
Hi
HTTP
assert_equal(expected, response)
connection << <<~HTTP.crlf_lines
GET / HTTP/1.1
Upgrade: echo
Connection: upgrade
HTTP
snooze
response = connection.readpartial(8192)
snooze
assert !connection.eof?
expected = <<~HTTP.crlf_lines
HTTP/1.1 101 Switching Protocols
Upgrade: echo
Connection: Upgrade
HTTP
assert_equal(expected, response)
assert !done
connection << 'foo'
assert_equal 'foo', connection.readpartial(8192)
connection << 'bar'
assert_equal 'bar', connection.readpartial(8192)
connection.close
assert !done
sleep 0.01
assert done
end
def test_big_download
chunk_size = 1000
chunk_count = 1000
chunk = '*' * chunk_size
@server, connection = spin_server do |req|
req.send_headers
chunk_count.times do |i|
req << chunk
snooze
end
req.finish
req.adapter.close
end
response = +''
count = 0
connection << "GET / HTTP/1.1\r\n\r\n"
while (data = connection.read(chunk_size))
response << data
count += 1
snooze
end
chunks = "#{chunk_size.to_s(16)}\n#{'*' * chunk_size}\n" * chunk_count
expected = <<~HTTP.crlf_lines
HTTP/1.1 200
Transfer-Encoding: chunked
#{chunks}0
HTTP
assert_equal expected, response
assert count >= chunk_count
end
end
================================================
FILE: test/test_request.rb
================================================
# frozen_string_literal: true
require_relative 'helper'
require 'tipi'
class String
def http_lines
gsub "\n", "\r\n"
end
end
class RequestHeadersTest < Minitest::Test
def teardown
@server&.interrupt if @server&.alive?
snooze
super
end
def spin_server(opts = {}, &handler)
server_connection, client_connection = IO.server_client_mockup
coproc = spin do
Tipi.client_loop(server_connection, opts, &handler)
end
[coproc, client_connection, server_connection]
end
def test_request_headers
req = nil
@server, connection = spin_server do |r|
req = r
req.respond('Hello, world!')
end
connection << "GET /titi HTTP/1.1\r\nHost: blah.com\r\nFoo: bar\r\nhi: 1\r\nHi: 2\r\nhi: 3\r\n\r\n"
sleep 0.01
assert_kind_of Qeweney::Request, req
assert_equal 'blah.com', req.headers['host']
assert_equal 'bar', req.headers['foo']
assert_equal ['1', '2', '3'], req.headers['hi']
assert_equal 'GET', req.headers[':method']
assert_equal '/titi', req.headers[':path']
end
def test_request_host
req = nil
@server, connection = spin_server do |r|
req = r
req.respond('Hello, world!')
end
connection << "GET /titi HTTP/1.1\nHost: blah.com\nFoo: bar\nhi: 1\nHi: 2\nhi: 3\n\n"
sleep 0.01
assert_equal 'blah.com', req.host
end
def test_request_connection
req = nil
@server, connection = spin_server do |r|
req = r
req.respond('Hello, world!')
end
connection << "GET /titi HTTP/1.1\nConnection: keep-alive\nFoo: bar\nhi: 1\nHi: 2\nhi: 3\n\n"
sleep 0.01
assert_equal 'keep-alive', req.connection
end
def test_request_upgrade_protocol
req = nil
@server, connection = spin_server do |r|
req = r
req.respond('Hello, world!')
end
connection << "GET /titi HTTP/1.1\nConnection: upgrade\nUpgrade: foobar\n\n"
sleep 0.01
assert_equal 'foobar', req.upgrade_protocol
end
end
================================================
FILE: tipi.gemspec
================================================
require_relative './lib/tipi/version'
Gem::Specification.new do |s|
s.name = 'tipi'
s.version = Tipi::VERSION
s.licenses = ['MIT']
s.summary = 'Tipi - the All-in-one Web Server for Ruby Apps'
s.author = 'Sharon Rosner'
s.email = 'sharon@noteflakes.com'
s.files = `git ls-files`.split
s.homepage = 'http://github.com/digital-fabric/tipi'
s.metadata = {
"source_code_uri" => "https://github.com/digital-fabric/tipi",
"documentation_uri" => "https://www.rubydoc.info/gems/tipi",
"homepage_uri" => "https://github.com/digital-fabric/tipi",
"changelog_uri" => "https://github.com/digital-fabric/tipi/blob/master/CHANGELOG.md"
}
s.rdoc_options = ["--title", "tipi", "--main", "README.md"]
s.extra_rdoc_files = ["README.md"]
s.require_paths = ["lib"]
s.required_ruby_version = '>= 3.2'
s.executables = ['tipi']
s.add_runtime_dependency 'base64', '~>0.3'
s.add_runtime_dependency 'mutex_m', '~>0.3'
s.add_runtime_dependency 'polyphony', '~>1.4'
s.add_runtime_dependency 'ever', '~>0.2'
s.add_runtime_dependency 'qeweney', '~>0.24'
s.add_runtime_dependency 'extralite', '~>2.13'
s.add_runtime_dependency 'h1p', '~>1.1'
s.add_runtime_dependency 'http-2', '~>1.1'
s.add_runtime_dependency 'rack', '>=2.0.8', '<3.3.0'
s.add_runtime_dependency 'websocket', '~>1.2.11'
s.add_runtime_dependency 'acme-client', '~>2.0.26'
s.add_runtime_dependency 'localhost', '~>1.6.0'
# for digital fabric
s.add_runtime_dependency 'msgpack', '~>1.8.0'
s.add_development_dependency 'logger', '~>1.7'
s.add_development_dependency 'ostruct', '~>0.3'
s.add_development_dependency 'rake', '~>13.3.0'
s.add_development_dependency 'minitest', '~>5.26.0'
s.add_development_dependency 'simplecov', '~>0.22.0'
s.add_development_dependency 'memory_profiler', '~>1.1.0'
s.add_development_dependency 'cuba', '~>4.0.3'
end
gitextract_pwtx6b4h/ ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ └── test.yml ├── .gitignore ├── CHANGELOG.md ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── TODO.md ├── benchmarks/ │ └── bm_http1_parser.rb ├── bin/ │ ├── benchmark │ ├── h1pd │ └── tipi ├── df/ │ ├── agent.rb │ ├── etc_benchmark.rb │ ├── multi_agent_supervisor.rb │ ├── multi_client.rb │ ├── routing_benchmark.rb │ ├── sample_agent.rb │ ├── server.rb │ ├── server_utils.rb │ ├── sse_page.html │ ├── stress.rb │ └── ws_page.html ├── docs/ │ └── README.md ├── examples/ │ ├── cuba.ru │ ├── full_service.rb │ ├── hanami-api.ru │ ├── hello.rb │ ├── hello.ru │ ├── http1_parser.rb │ ├── http_request_ws_server.rb │ ├── http_server.js │ ├── http_server.rb │ ├── http_server_forked.rb │ ├── http_server_form.rb │ ├── http_server_graceful.rb │ ├── http_server_routes.rb │ ├── http_server_simple.rb │ ├── http_server_static.rb │ ├── http_server_throttled.rb │ ├── http_server_throttled_accept.rb │ ├── http_server_timeout.rb │ ├── http_unix_socket_server.rb │ ├── http_ws_server.rb │ ├── https_server.rb │ ├── https_server_forked.rb │ ├── https_wss_server.rb │ ├── rack_server.rb │ ├── rack_server_forked.rb │ ├── rack_server_https.rb │ ├── rack_server_https_forked.rb │ ├── routing_server.rb │ ├── servername_cb.rb │ ├── source.rb │ ├── streaming.rb │ ├── websocket_client.rb │ ├── websocket_demo.rb │ ├── websocket_secure_server.rb │ ├── websocket_server.rb │ ├── ws_page.html │ ├── wss_page.html │ └── zlib-bench.rb ├── lib/ │ ├── tipi/ │ │ ├── acme.rb │ │ ├── cli.rb │ │ ├── config_dsl.rb │ │ ├── configuration.rb │ │ ├── controller/ │ │ │ ├── bare_polyphony.rb │ │ │ ├── bare_stock.rb │ │ │ ├── extensions.rb │ │ │ ├── stock_http1_adapter.rb │ │ │ ├── web_polyphony.rb │ │ │ └── web_stock.rb │ │ ├── controller.rb │ │ ├── digital_fabric/ │ │ │ ├── agent.rb │ │ │ ├── agent_proxy.rb │ │ │ ├── executive/ │ │ │ │ └── index.html │ │ │ ├── executive.rb │ │ │ ├── protocol.rb │ │ │ ├── request_adapter.rb │ │ │ └── service.rb │ │ ├── digital_fabric.rb │ │ ├── handler.rb │ │ ├── http1_adapter.rb │ │ ├── http2_adapter.rb │ │ ├── http2_stream.rb │ │ ├── rack_adapter.rb │ │ ├── response_extensions.rb │ │ ├── supervisor.rb │ │ ├── version.rb │ │ └── websocket.rb │ └── tipi.rb ├── test/ │ ├── coverage.rb │ ├── eg.rb │ ├── helper.rb │ ├── run.rb │ ├── test_http_server.rb │ └── test_request.rb └── tipi.gemspec
SYMBOL INDEX (519 symbols across 51 files)
FILE: benchmarks/bm_http1_parser.rb
function measure_time_and_allocs (line 7) | def measure_time_and_allocs
function object_count (line 21) | def object_count
function benchmark_other_http1_parser (line 26) | def benchmark_other_http1_parser(iterations)
function benchmark_tipi_http1_parser (line 54) | def benchmark_tipi_http1_parser(iterations)
function fork_benchmark (line 70) | def fork_benchmark(method, iterations)
FILE: df/agent.rb
class SampleAgent (line 11) | class SampleAgent < DigitalFabric::Agent
method initialize (line 12) | def initialize(id, server_url)
method http_request (line 18) | def http_request(req)
method streaming_http_request (line 25) | def streaming_http_request(req)
method form_http_request (line 42) | def form_http_request(req)
method do_some_activity (line 48) | def do_some_activity
method generate_data (line 52) | def generate_data(length)
FILE: df/etc_benchmark.rb
function generate (line 5) | def generate
FILE: df/multi_agent_supervisor.rb
class AgentManager (line 12) | class AgentManager
method initialize (line 13) | def initialize
method process_pending_action (line 19) | def process_pending_action
method start_agent (line 29) | def start_agent(spec)
method stop_agent (line 42) | def stop_agent(spec)
method update (line 50) | def update
method run (line 63) | def run
class RealityAgentManager (line 68) | class RealityAgentManager < AgentManager
method agent_specs (line 69) | def agent_specs
method launch_agent_from_spec (line 73) | def launch_agent_from_spec(spec)
FILE: df/multi_client.rb
class Client (line 7) | class Client
method initialize (line 8) | def initialize(id, host, port, http_host, interval)
method run (line 17) | def run
method connect (line 24) | def connect
method issue_requests (line 36) | def issue_requests
method do_request (line 51) | def do_request
method wait_for_response (line 62) | def wait_for_response
function spin_client (line 70) | def spin_client(id, host)
FILE: df/routing_benchmark.rb
class FakeAgent (line 7) | class FakeAgent
method initialize (line 8) | def initialize(idx)
function setup_df_service_with_agents (line 13) | def setup_df_service_with_agents(agent_count)
function benchmark_route_compilation (line 21) | def benchmark_route_compilation(agent_count, iterations)
class FauxRequest (line 29) | class FauxRequest
method initialize (line 30) | def initialize(agent_count)
method headers (line 34) | def headers
function benchmark_find_agent (line 39) | def benchmark_find_agent(agent_count, iterations)
function benchmark (line 50) | def benchmark
FILE: df/sample_agent.rb
class SampleAgent (line 11) | class SampleAgent < DigitalFabric::Agent
method http_request (line 15) | def http_request(req)
method ws_request (line 52) | def ws_request(req)
method stream_sse_response (line 62) | def stream_sse_response(req)
FILE: df/server_utils.rb
function log (line 19) | def log(msg, **ctx)
function listen_http (line 32) | def listen_http
function listen_https (line 61) | def listen_https
function listen_unix (line 115) | def listen_unix
function listen_df (line 136) | def listen_df
FILE: df/stress.rb
function monitor_process (line 9) | def monitor_process(cmd)
FILE: examples/http1_parser.rb
type ::Kernel (line 8) | module ::Kernel
function trace (line 9) | def trace(*args)
function format_trace (line 13) | def format_trace(args)
FILE: examples/http_request_ws_server.rb
function ws_handler (line 7) | def ws_handler(conn)
FILE: examples/http_server.js
constant MSG (line 7) | const MSG = 'Hello World';
FILE: examples/http_server_timeout.rb
function timeout_handler (line 11) | def timeout_handler(timeout, &handler)
FILE: examples/http_ws_server.rb
function ws_handler (line 7) | def ws_handler(conn)
FILE: examples/https_wss_server.rb
function ws_handler (line 8) | def ws_handler(conn)
FILE: examples/websocket_client.rb
class WebsocketClient (line 9) | class WebsocketClient
method initialize (line 10) | def initialize(url, headers = {})
method do_handshake (line 15) | def do_handshake(url, headers)
method receive (line 28) | def receive
method send (line 36) | def send(data)
method close (line 46) | def close
FILE: examples/websocket_demo.rb
class WebsocketClient (line 7) | class WebsocketClient
method initialize (line 8) | def initialize(url, headers = {})
method do_handshake (line 13) | def do_handshake(url, headers)
method receive (line 26) | def receive
method send (line 37) | def send(data)
method close (line 47) | def close
FILE: examples/websocket_secure_server.rb
function ws_handler (line 7) | def ws_handler(conn)
FILE: examples/websocket_server.rb
function ws_handler (line 7) | def ws_handler(conn)
FILE: examples/zlib-bench.rb
function create_files (line 10) | def create_files
FILE: lib/tipi.rb
class Qeweney::Request (line 13) | class Qeweney::Request
type Tipi (line 17) | module Tipi
function serve (line 22) | def serve(host, port, opts = {}, &handler)
function listen (line 30) | def listen(host, port, opts = {})
function accept_loop (line 39) | def accept_loop(server, opts, &handler)
function client_loop (line 47) | def client_loop(client, opts, &handler)
function protocol_adapter (line 57) | def protocol_adapter(socket, opts)
function route (line 64) | def route(&block)
FILE: lib/tipi/acme.rb
type Tipi (line 7) | module Tipi
type ACME (line 8) | module ACME
class Error (line 9) | class Error < StandardError
class CertificateManager (line 12) | class CertificateManager
method initialize (line 13) | def initialize(master_ctx:, store:, challenge_handler:, valid_hosts:)
method challenge_routing_app (line 26) | def challenge_routing_app(app)
method setup_sni_callback (line 38) | def setup_sni_callback
method get_ctx (line 42) | def get_ctx(name)
method wait_for_ctx (line 65) | def wait_for_ctx(state)
method run (line 78) | def run
method get_context (line 89) | def get_context(name)
method setup_context (line 93) | def setup_context(name)
method provision_context (line 99) | def provision_context(name)
method transfer_ctx_settings (line 110) | def transfer_ctx_settings(ctx)
method parse_certificate (line 118) | def parse_certificate(certificate)
method get_expired_stamp (line 124) | def get_expired_stamp(certificate)
method get_certificate (line 130) | def get_certificate(name)
method localhost_context (line 139) | def localhost_context
method private_key (line 144) | def private_key
method acme_client (line 150) | def acme_client
method setup_acme_client (line 154) | def setup_acme_client
method provision_certificate (line 166) | def provision_certificate(name)
class HTTPChallengeHandler (line 205) | class HTTPChallengeHandler
method initialize (line 206) | def initialize
method add (line 210) | def add(challenge)
method remove (line 215) | def remove(challenge)
method call (line 220) | def call(req)
class CertificateStore (line 231) | class CertificateStore
method set (line 232) | def set(name, private_key:, certificate:, expired_stamp:)
method get (line 236) | def get(name)
class InMemoryCertificateStore (line 241) | class InMemoryCertificateStore
method initialize (line 242) | def initialize
method set (line 246) | def set(name, private_key:, certificate:, expired_stamp:)
method get (line 254) | def get(name)
class SQLiteCertificateStore (line 266) | class SQLiteCertificateStore
method initialize (line 269) | def initialize(path)
method set (line 283) | def set(name, private_key:, certificate:, expired_stamp:)
method get (line 292) | def get(name)
method remove_expired_certificates (line 309) | def remove_expired_certificates
FILE: lib/tipi/cli.rb
type Tipi (line 8) | module Tipi
function opts_from_argv (line 18) | def self.opts_from_argv(argv)
function parse_listen_spec (line 51) | def self.parse_listen_spec(type, spec)
function str_to_native_type (line 55) | def self.str_to_native_type(str)
function verify_path (line 64) | def self.verify_path(path)
type CLI (line 71) | module CLI
function start (line 82) | def self.start(argv = ARGV.dup)
function display_banner (line 89) | def self.display_banner
FILE: lib/tipi/config_dsl.rb
type Tipi (line 3) | module Tipi
type Configuration (line 4) | module Configuration
class Interpreter (line 5) | class Interpreter
method initialize (line 8) | def initialize(assembler)
method gzip_response (line 12) | def gzip_response
method log (line 16) | def log(out)
method error (line 20) | def error(&block)
method match (line 24) | def match(pattern, &block)
class Assembler (line 29) | class Assembler
method from_source (line 30) | def self.from_source(code)
method from_source (line 34) | def from_source(code)
method new_frame (line 48) | def new_frame
method add_frame (line 55) | def add_frame(&block)
method wrap_current_frame (line 63) | def wrap_current_frame(head)
method emit (line 71) | def emit(code)
method emit_prelude (line 75) | def emit_prelude(code)
method emit_exception_handler (line 79) | def emit_exception_handler(&block)
method emit_block (line 84) | def emit_block(conditional, &block)
method add_app_proc (line 91) | def add_app_proc(proc)
method assemble_frame (line 97) | def assemble_frame(frame)
method assemble_app_proc (line 115) | def assemble_app_proc(frame)
method emit_code (line 126) | def emit_code(lines, code, indent)
method indent_line (line 136) | def indent_line(code, indent)
function assemble (line 144) | def assemble(code)
FILE: lib/tipi/configuration.rb
type Tipi (line 5) | module Tipi
type Configuration (line 6) | module Configuration
function supervise_config (line 8) | def supervise_config
function run (line 16) | def run(config)
function start_listeners (line 21) | def start_listeners(config)
function simple_supervise (line 26) | def simple_supervise(config)
function forked_supervise (line 33) | def forked_supervise(config)
function setup_virtual_hosts (line 40) | def setup_virtual_hosts(config)
function start_acceptors (line 46) | def start_acceptors(config, virtual_hosts)
FILE: lib/tipi/controller/bare_stock.rb
type Tipi (line 3) | module Tipi
type Apps (line 4) | module Apps
type Bare (line 5) | module Bare
function start (line 6) | def start(opts)
FILE: lib/tipi/controller/extensions.rb
type Kernel (line 5) | module Kernel
function run (line 6) | def run(app = nil, &block)
type Tipi (line 11) | module Tipi
function app (line 15) | def app
function run_sites (line 21) | def run_sites(site_map)
FILE: lib/tipi/controller/stock_http1_adapter.rb
type Tipi (line 5) | module Tipi
class StockHTTP1Adapter (line 6) | class StockHTTP1Adapter < HTTP1Adapter
method initialize (line 7) | def initialize(conn, opts)
method each (line 12) | def each(&block)
FILE: lib/tipi/controller/web_polyphony.rb
type Tipi (line 7) | module Tipi
class Controller (line 8) | class Controller
method initialize (line 9) | def initialize(opts)
method run (line 17) | def run
method supervise_workers (line 26) | def supervise_workers(worker_count)
method run_worker (line 52) | def run_worker
method prepare_service (line 66) | def prepare_service
method start_app (line 76) | def start_app
method rack_service (line 84) | def rack_service
method tipi_service (line 90) | def tipi_service
method static_service (line 98) | def static_service
method web_service (line 111) | def web_service(app)
method prepare_listener (line 117) | def prepare_listener(spec, app)
method prepare_http_listener (line 152) | def prepare_http_listener(port, app)
method prepare_https_listener (line 164) | def prepare_https_listener(host, port, app)
method prepare_localhost_https_listener (line 172) | def prepare_localhost_https_listener(port, app)
method prepare_full_service_listeners (line 189) | def prepare_full_service_listeners(host, http_port, https_port, app)
method http_redirect_app (line 223) | def http_redirect_app(https_port)
method find_path (line 234) | def find_path(base, path)
method spin_accept_loop (line 253) | def spin_accept_loop(name, port, &block)
method spin_connection_handler (line 269) | def spin_connection_handler(name, socket, block)
method finalize_listener (line 280) | def finalize_listener(server)
method gracefully_terminate_conections (line 290) | def gracefully_terminate_conections(fiber)
method add_connection_headers (line 298) | def add_connection_headers(app)
method ssl_accept (line 308) | def ssl_accept(client)
method start_https_connection_fiber (line 318) | def start_https_connection_fiber(socket, ctx, thread_pool, app)
method create_certificate_store (line 341) | def create_certificate_store
method start_server (line 346) | def start_server(service)
FILE: lib/tipi/controller/web_stock.rb
type Tipi (line 10) | module Tipi
class Listener (line 11) | class Listener
method initialize (line 12) | def initialize(server, &handler)
method accept (line 17) | def accept
class Connection (line 23) | class Connection
method io_ready (line 24) | def io_ready
class HTTP1Connection (line 29) | class HTTP1Connection < Connection
method initialize (line 32) | def initialize(io, evloop, &app)
method setup_read_request (line 40) | def setup_read_request
method on_headers_complete (line 46) | def on_headers_complete(headers)
method normalize_headers (line 56) | def normalize_headers(headers)
method scheme_from_connection (line 69) | def scheme_from_connection
method on_body (line 73) | def on_body(chunk)
method on_message_complete (line 77) | def on_message_complete
method io_ready (line 81) | def io_ready
method handle_read_request (line 89) | def handle_read_request
method watch_io (line 112) | def watch_io(rw)
method close_io (line 117) | def close_io
method handle_request (line 121) | def handle_request
method respond (line 138) | def respond(request, body, headers)
method send_headers (line 155) | def send_headers(request, headers, empty_response: false, chunked: t...
method http1_1? (line 161) | def http1_1?(request)
method send_chunk (line 172) | def send_chunk(request, chunk, done: false)
method finish (line 185) | def finish(request)
method format_headers (line 198) | def format_headers(headers, body, chunked)
method format_status_line (line 211) | def format_status_line(body, status, chunked)
method empty_status_line (line 219) | def empty_status_line(status)
method with_body_status_line (line 227) | def with_body_status_line(status, body, chunked)
method collect_header_lines (line 235) | def collect_header_lines(lines, key, value)
method handle_write (line 243) | def handle_write(data = nil)
class Controller (line 267) | class Controller
method initialize (line 268) | def initialize(opts)
method run (line 276) | def run
method supervise_workers (line 285) | def supervise_workers(worker_count)
method run_worker (line 307) | def run_worker
method run_evloop (line 318) | def run_evloop
method prepare_service (line 337) | def prepare_service
method start_app (line 347) | def start_app
method rack_service (line 355) | def rack_service
method tipi_service (line 361) | def tipi_service
method static_service (line 371) | def static_service
method web_service (line 385) | def web_service(app)
method prepare_listener (line 391) | def prepare_listener(spec, app)
method prepare_http_listener (line 426) | def prepare_http_listener(port, app)
method start_client (line 436) | def start_client(socket, &app)
method prepare_https_listener (line 443) | def prepare_https_listener(host, port, app)
method prepare_localhost_https_listener (line 451) | def prepare_localhost_https_listener(port, app)
method prepare_full_service_listeners (line 468) | def prepare_full_service_listeners(host, http_port, https_port, app)
method find_path (line 505) | def find_path(base, path)
method start_listener (line 524) | def start_listener(name, port, &block)
method spin_accept_loop (line 538) | def spin_accept_loop(name, port, &block)
method spin_connection_handler (line 554) | def spin_connection_handler(name, socket, block)
method finalize_listener (line 565) | def finalize_listener(server)
method gracefully_terminate_conections (line 575) | def gracefully_terminate_conections(fiber)
method add_connection_headers (line 583) | def add_connection_headers(app)
method ssl_accept (line 593) | def ssl_accept(client)
method start_https_connection_fiber (line 603) | def start_https_connection_fiber(socket, ctx, thread_pool, app)
method create_certificate_store (line 626) | def create_certificate_store
method start_server (line 631) | def start_server(service)
FILE: lib/tipi/digital_fabric.rb
type DigitalFabric (line 1) | module DigitalFabric
FILE: lib/tipi/digital_fabric/agent.rb
type DigitalFabric (line 10) | module DigitalFabric
class Agent (line 11) | class Agent
method initialize (line 12) | def initialize(server_url, route, token)
class TimeoutError (line 21) | class TimeoutError < RuntimeError
class GracefulShutdown (line 24) | class GracefulShutdown < RuntimeError
method run (line 29) | def run
method connect_and_process_incoming_requests (line 41) | def connect_and_process_incoming_requests
method connect_to_server (line 56) | def connect_to_server
method df_upgrade (line 76) | def df_upgrade
method mount_point (line 84) | def mount_point
method log (line 94) | def log(msg)
method process_incoming_requests (line 98) | def process_incoming_requests
method keep_alive (line 107) | def keep_alive
method recv_df_message (line 122) | def recv_df_message(msg)
method send_df_message (line 140) | def send_df_message(msg)
method is_long_running_request_response? (line 152) | def is_long_running_request_response?(msg)
method recv_shutdown (line 161) | def recv_shutdown
method recv_http_request (line 168) | def recv_http_request(msg)
method prepare_http_request (line 185) | def prepare_http_request(msg)
method recv_http_request_body (line 194) | def recv_http_request_body(msg)
method get_http_request_body (line 201) | def get_http_request_body(id, limit)
method recv_ws_request (line 206) | def recv_ws_request(msg)
method http_request (line 221) | def http_request(req)
method ws_request (line 226) | def ws_request(req)
FILE: lib/tipi/digital_fabric/agent_proxy.rb
type DigitalFabric (line 7) | module DigitalFabric
class AgentProxy (line 8) | class AgentProxy
method initialize (line 9) | def initialize(service, req)
method current_request_count (line 21) | def current_request_count
class TimeoutError (line 25) | class TimeoutError < RuntimeError
class GracefulShutdown (line 28) | class GracefulShutdown < RuntimeError
method run (line 31) | def run
method process_incoming_messages (line 45) | def process_incoming_messages(shutdown = false)
method unmount (line 56) | def unmount
method send_shutdown (line 63) | def send_shutdown
method keep_alive (line 68) | def keep_alive
method route (line 79) | def route
method recv_df_message (line 92) | def recv_df_message(message)
method send_df_message (line 114) | def send_df_message(message)
method register_request_fiber (line 123) | def register_request_fiber
method unregister_request_fiber (line 129) | def unregister_request_fiber(id)
method with_request (line 133) | def with_request
method http_request (line 143) | def http_request(req)
method http_request_send_error_response (line 171) | def http_request_send_error_response(error)
method http_request_message (line 179) | def http_request_message(id, req, kind, message)
method send_transfer_count (line 195) | def send_transfer_count(key, rx, tx)
method handle_stats_request (line 199) | def handle_stats_request(id)
method http_custom_upgrade (line 206) | def http_custom_upgrade(id, req, headers)
method http_custom_upgrade_message (line 226) | def http_custom_upgrade_message(conn, message)
method http_response (line 239) | def http_response(id, req, body, headers, complete, transfer_count_key)
method http_get_request_body (line 261) | def http_get_request_body(id, req, limit)
method http_upgrade (line 276) | def http_upgrade(req, protocol)
method handle_websocket_upgrade (line 288) | def handle_websocket_upgrade(req)
method run_websocket_connection (line 310) | def run_websocket_connection(id, websocket)
FILE: lib/tipi/digital_fabric/executive.rb
type DigitalFabric (line 6) | module DigitalFabric
class Executive (line 8) | class Executive
method initialize (line 13) | def initialize(service, route = { path: '/executive' })
method current_request_count (line 22) | def current_request_count
method http_request (line 26) | def http_request(req)
method stream_stats (line 47) | def stream_stats(req)
method format_sse_event (line 60) | def format_sse_event(data)
method update_service_stats (line 64) | def update_service_stats
method machine_stats (line 77) | def machine_stats
FILE: lib/tipi/digital_fabric/protocol.rb
type DigitalFabric (line 3) | module DigitalFabric
type Protocol (line 4) | module Protocol
type Attribute (line 31) | module Attribute
type HttpRequest (line 35) | module HttpRequest
type HttpResponse (line 41) | module HttpResponse
type HttpUpgrade (line 48) | module HttpUpgrade
type HttpGetRequestBody (line 52) | module HttpGetRequestBody
type HttpRequestBody (line 56) | module HttpRequestBody
type ConnectionData (line 61) | module ConnectionData
type WS (line 65) | module WS
type TransferCount (line 70) | module TransferCount
type Stats (line 76) | module Stats
function ping (line 82) | def ping
function shutdown (line 86) | def shutdown
function unmount (line 90) | def unmount
function df_upgrade_response (line 101) | def df_upgrade_response
function http_request (line 105) | def http_request(id, headers, buffered_chunk, complete)
function http_response (line 109) | def http_response(id, body, headers, complete, transfer_count_key = ...
function http_upgrade (line 113) | def http_upgrade(id, headers)
function http_get_request_body (line 117) | def http_get_request_body(id, limit = nil)
function http_request_body (line 121) | def http_request_body(id, body, complete)
function connection_data (line 125) | def connection_data(id, data)
function connection_close (line 129) | def connection_close(id)
function ws_request (line 133) | def ws_request(id, headers)
function ws_response (line 137) | def ws_response(id, headers)
function ws_data (line 141) | def ws_data(id, data)
function ws_close (line 145) | def ws_close(id)
function transfer_count (line 149) | def transfer_count(key, rx, tx)
function stats_request (line 153) | def stats_request(id)
function stats_response (line 157) | def stats_response(id, stats)
FILE: lib/tipi/digital_fabric/request_adapter.rb
type DigitalFabric (line 5) | module DigitalFabric
class RequestAdapter (line 6) | class RequestAdapter
method initialize (line 7) | def initialize(agent, msg)
method protocol (line 12) | def protocol
method get_body_chunk (line 16) | def get_body_chunk(request)
method respond (line 20) | def respond(request, body, headers)
method send_headers (line 26) | def send_headers(request, headers, opts = {})
method send_chunk (line 32) | def send_chunk(request, body, done: )
method finish (line 38) | def finish(request)
FILE: lib/tipi/digital_fabric/service.rb
type DigitalFabric (line 7) | module DigitalFabric
class Service (line 8) | class Service
method initialize (line 12) | def initialize(token: )
method calculate_stats (line 31) | def calculate_stats
method pid_cpu_and_rss (line 84) | def pid_cpu_and_rss(pid)
method get_stats (line 94) | def get_stats
method incr_connection_count (line 98) | def incr_connection_count
method decr_connection_count (line 102) | def decr_connection_count
method total_request_count (line 108) | def total_request_count
method record_latency_measurement (line 118) | def record_latency_measurement(latency, req)
method http_request (line 127) | def http_request(req, allow_df_upgrade = false)
method inject_request_headers (line 156) | def inject_request_headers(req)
method upgrade_request (line 163) | def upgrade_request(req, allow_df_upgrade)
method df_upgrade (line 182) | def df_upgrade(req)
method mount (line 195) | def mount(route, agent)
method unmount (line 204) | def unmount(agent)
method find_agent (line 215) | def find_agent(req)
method compile_agent_routes (line 229) | def compile_agent_routes
method path_regexp (line 240) | def path_regexp(path)
method graceful_shutdown (line 244) | def graceful_shutdown
FILE: lib/tipi/handler.rb
type Tipi (line 7) | module Tipi
class DefaultHandler (line 8) | class DefaultHandler
method initialize (line 9) | def initialize(config)
method call (line 16) | def call(socket)
method protocol_adapter (line 27) | def protocol_adapter(socket, opts)
FILE: lib/tipi/http1_adapter.rb
type Tipi (line 8) | module Tipi
class HTTP1Adapter (line 10) | class HTTP1Adapter
method initialize (line 14) | def initialize(conn, opts)
method each (line 21) | def each(&block)
method handle_request (line 36) | def handle_request(headers, &block)
method persistent_connection? (line 56) | def persistent_connection?(headers)
method finalize_client_loop (line 65) | def finalize_client_loop
method get_body_chunk (line 74) | def get_body_chunk(request, buffered_only = false)
method get_body (line 78) | def get_body(request)
method complete? (line 82) | def complete?(request)
method protocol (line 86) | def protocol
method upgrade_connection (line 112) | def upgrade_connection(headers, &block)
method upgrade_with_handler (line 124) | def upgrade_with_handler(handler, headers)
method upgrade_to_http2 (line 130) | def upgrade_to_http2(headers, &block)
method http2_upgraded_headers (line 140) | def http2_upgraded_headers(headers)
method websocket_connection (line 147) | def websocket_connection(request)
method scheme_from_connection (line 151) | def scheme_from_connection
method respond (line 164) | def respond(request, body, headers)
method respond_from_io (line 171) | def respond_from_io(request, io, headers, chunk_size = 2**14)
method send_headers (line 194) | def send_headers(request, headers, empty_response: false, chunked: t...
method http1_1? (line 200) | def http1_1?(request)
method send_chunk (line 211) | def send_chunk(request, chunk, done: false)
method send_chunk_from_io (line 226) | def send_chunk_from_io(request, io, r, w, chunk_size)
method finish (line 243) | def finish(request)
method close (line 248) | def close
method format_headers (line 263) | def format_headers(headers, body, chunked)
method format_status_line (line 276) | def format_status_line(body, status, chunked)
method empty_status_line (line 284) | def empty_status_line(status)
method with_body_status_line (line 292) | def with_body_status_line(status, body, chunked)
method collect_header_lines (line 300) | def collect_header_lines(lines, key, value)
FILE: lib/tipi/http2_adapter.rb
class HTTP2::Stream (line 7) | class HTTP2::Stream
method end_stream? (line 8) | def end_stream?(frame)
type Tipi (line 17) | module Tipi
class HTTP2Adapter (line 19) | class HTTP2Adapter
method upgrade_each (line 20) | def self.upgrade_each(socket, opts, headers, body, &block)
method initialize (line 25) | def initialize(conn, opts, upgrade_headers = nil, upgrade_body = nil)
method send_frame (line 40) | def send_frame(data)
method upgrade (line 58) | def upgrade
method each (line 68) | def each(&block)
method get_rx_count (line 82) | def get_rx_count
method get_tx_count (line 88) | def get_tx_count
method start_stream (line 94) | def start_stream(stream, &block)
method finalize_client_loop (line 100) | def finalize_client_loop
method close (line 107) | def close
method set_request_for_transfer_count (line 112) | def set_request_for_transfer_count(request)
method unset_request_for_transfer_count (line 116) | def unset_request_for_transfer_count(request)
FILE: lib/tipi/http2_stream.rb
type Tipi (line 6) | module Tipi
class HTTP2StreamHandler (line 8) | class HTTP2StreamHandler
method initialize (line 11) | def initialize(adapter, stream, conn, first, &block)
method run (line 35) | def run(&block)
method on_headers (line 48) | def on_headers(headers)
method on_data (line 59) | def on_data(data)
method on_half_close (line 66) | def on_half_close
method protocol (line 71) | def protocol
method with_transfer_count (line 75) | def with_transfer_count(request)
method get_body_chunk (line 82) | def get_body_chunk(request, buffered_only = false)
method get_body (line 96) | def get_body(request)
method complete? (line 111) | def complete?(request)
method respond (line 116) | def respond(request, body, headers)
method respond_from_io (line 127) | def respond_from_io(request, io, headers, chunk_size = 2**16)
method transform_headers (line 140) | def transform_headers(headers)
method send_headers (line 150) | def send_headers(request, headers, empty_response: false)
method send_chunk (line 163) | def send_chunk(request, chunk, done: false)
method finish (line 177) | def finish(request)
method stop (line 190) | def stop
method normalize_status_header (line 199) | def normalize_status_header(headers, default_status = Qeweney::Statu...
FILE: lib/tipi/rack_adapter.rb
type Tipi (line 5) | module Tipi
type RackAdapter (line 6) | module RackAdapter
function run (line 8) | def run(app)
function load (line 12) | def load(path)
function env (line 17) | def env(request)
function respond (line 21) | def respond(request, (status_code, headers, body))
FILE: lib/tipi/response_extensions.rb
type Tipi (line 5) | module Tipi
type ResponseExtensions (line 6) | module ResponseExtensions
function serve_io (line 9) | def serve_io(io, opts)
FILE: lib/tipi/supervisor.rb
type Tipi (line 6) | module Tipi
type Supervisor (line 7) | module Supervisor
function run (line 9) | def run(opts)
function start_controller_watcher (line 16) | def start_controller_watcher
function controller_cmd (line 35) | def controller_cmd
function supervise_loop (line 43) | def supervise_loop
function replace_controller (line 63) | def replace_controller
FILE: lib/tipi/version.rb
type Tipi (line 3) | module Tipi
FILE: lib/tipi/websocket.rb
type Tipi (line 6) | module Tipi
class Websocket (line 8) | class Websocket
method handler (line 9) | def self.handler(&block)
method initialize (line 17) | def initialize(conn, headers)
method recv (line 24) | def recv
method recv_loop (line 39) | def recv_loop
method send (line 54) | def send(data)
method close (line 62) | def close
FILE: test/coverage.rb
function whitespace_line? (line 8) | def whitespace_line?(line)
type Coverage (line 13) | module Coverage
function relevant_lines_for_filename (line 20) | def relevant_lines_for_filename(filename)
function start (line 25) | def start
function result (line 39) | def result
FILE: test/eg.rb
type Kernel (line 3) | module Kernel
function eg (line 7) | def eg(hash)
FILE: test/helper.rb
class Minitest::Test (line 16) | class Minitest::Test
method setup (line 17) | def setup
method teardown (line 26) | def teardown
type Kernel (line 41) | module Kernel
function capture_exception (line 42) | def capture_exception
function trace (line 48) | def trace(*args)
function format_trace (line 52) | def format_trace(args)
class IO (line 65) | class IO
method server_client_mockup (line 67) | def self.server_client_mockup
method mockup_connection (line 77) | def self.mockup_connection(input, output, output2)
type Minitest::Assertions (line 93) | module Minitest::Assertions
function assert_in_range (line 94) | def assert_in_range exp_range, act
FILE: test/test_http_server.rb
class String (line 6) | class String
method crlf_lines (line 7) | def crlf_lines
class HTTP1ServerTest (line 12) | class HTTP1ServerTest < Minitest::Test
method teardown (line 13) | def teardown
method spin_server (line 19) | def spin_server(opts = {}, &handler)
method test_that_server_uses_content_length_in_http_1_0 (line 27) | def test_that_server_uses_content_length_in_http_1_0
method test_that_server_uses_chunked_encoding_in_http_1_1 (line 45) | def test_that_server_uses_chunked_encoding_in_http_1_1
method test_that_server_maintains_connection_when_using_keep_alives (line 63) | def test_that_server_maintains_connection_when_using_keep_alives
method test_pipelining_client (line 93) | def test_pipelining_client
method test_body_chunks (line 118) | def test_body_chunks
method test_upgrade (line 171) | def test_upgrade
method test_big_download (line 241) | def test_big_download
FILE: test/test_request.rb
class String (line 6) | class String
method http_lines (line 7) | def http_lines
class RequestHeadersTest (line 12) | class RequestHeadersTest < Minitest::Test
method teardown (line 13) | def teardown
method spin_server (line 19) | def spin_server(opts = {}, &handler)
method test_request_headers (line 27) | def test_request_headers
method test_request_host (line 46) | def test_request_host
method test_request_connection (line 58) | def test_request_connection
method test_request_upgrade_protocol (line 70) | def test_request_upgrade_protocol
Condensed preview — 99 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (191K chars).
[
{
"path": ".github/FUNDING.yml",
"chars": 19,
"preview": "github: noteflakes\n"
},
{
"path": ".github/workflows/test.yml",
"chars": 627,
"preview": "name: Tests\n\non: [push, pull_request]\n\njobs:\n build:\n strategy:\n fail-fast: false\n matrix:\n os: [ub"
},
{
"path": ".gitignore",
"chars": 1260,
"preview": "*.gem\n*.rbc\n/.config\n/coverage/\n/InstalledFiles\n/pkg/\n/spec/reports/\n/spec/examples.txt\n/test/tmp/\n/test/version_tmp/\n/t"
},
{
"path": "CHANGELOG.md",
"chars": 4249,
"preview": "## 0.56 2025-10-21\n\n- Update localhost, acme-client, websocket, http-2, extralite, qeweney, rack, bundler dependencies ("
},
{
"path": "Gemfile",
"chars": 39,
"preview": "source 'https://rubygems.org'\n\ngemspec\n"
},
{
"path": "LICENSE",
"chars": 1070,
"preview": "MIT License\n\nCopyright (c) 2021 Sharon Rosner\n\nPermission is hereby granted, free of charge, to any person obtaining a c"
},
{
"path": "README.md",
"chars": 7226,
"preview": "<p align=\"center\"><img src=\"docs/tipi-logo.png\" /></p>\n\n# Tipi - the All-in-one Web Server for Ruby Apps\n\n[![Gem Version"
},
{
"path": "Rakefile",
"chars": 151,
"preview": "# frozen_string_literal: true\n\nrequire \"bundler/gem_tasks\"\nrequire \"rake/clean\"\n\ntask :default => [:test]\n\ntask :test do"
},
{
"path": "TODO.md",
"chars": 1083,
"preview": "## Rethink design\n\n- Remove DF code\n- Remove non-Polyphony code\n\n# Miscellaneous\n\n- Try using `TCP_DEFER_ACCEPT` with Po"
},
{
"path": "benchmarks/bm_http1_parser.rb",
"chars": 1982,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\n\nHTTP_REQUEST = \"GET /foo HTTP/1.1\\r\\nHost: example.com\\r\\nAccept"
},
{
"path": "bin/benchmark",
"chars": 865,
"preview": "#!/usr/bin/env ruby\n\nrequire 'bundler/setup'\nrequire 'polyphony'\n\ndef parse_latency(latency)\n m = latency.match(/^([\\d\\"
},
{
"path": "bin/h1pd",
"chars": 107,
"preview": "#!/usr/bin/env bash\n\nset -e\nrake compile\nruby test/test_http1_parser.rb\nruby benchmarks/bm_http1_parser.rb\n"
},
{
"path": "bin/tipi",
"chars": 106,
"preview": "#!/usr/bin/env ruby\n\nrequire 'bundler/setup'\nrequire 'tipi/cli'\n\ntrap('SIGINT') { exit }\n\nTipi::CLI.start\n"
},
{
"path": "df/agent.rb",
"chars": 1715,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'polyphony'\nrequire 'json'\nrequire 'tipi/digital_fabric/p"
},
{
"path": "df/etc_benchmark.rb",
"chars": 216,
"preview": "# frozen_string_literal: true\n\nrequire 'securerandom'\n\ndef generate\n SecureRandom.uuid\nend\n\ncount = 100000\n\nGC.disable\n"
},
{
"path": "df/multi_agent_supervisor.rb",
"chars": 1804,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'polyphony'\nrequire 'json'\n\nrequire 'fileutils'\nFileUtils"
},
{
"path": "df/multi_client.rb",
"chars": 1808,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'polyphony'\nrequire 'http/parser'\n\nclass Client\n def ini"
},
{
"path": "df/routing_benchmark.rb",
"chars": 1439,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'polyphony'\nrequire 'tipi/digital_fabric'\n\nclass FakeAgen"
},
{
"path": "df/sample_agent.rb",
"chars": 1904,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'polyphony'\nrequire 'json'\nrequire 'tipi/digital_fabric/p"
},
{
"path": "df/server.rb",
"chars": 547,
"preview": "# frozen_string_literal: true\n\nrequire_relative 'server_utils'\n\nlisteners = [\n listen_http,\n listen_https,\n listen_un"
},
{
"path": "df/server_utils.rb",
"chars": 5272,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'tipi/digital_fabric'\nrequire 'tipi/digita"
},
{
"path": "df/sse_page.html",
"chars": 702,
"preview": "<!doctype html>\n<html lang=\"en\">\n<head>\n <title>SSE Client</title>\n</head>\n<body>\n <h1>SSE Client</h1>\n <script>\n "
},
{
"path": "df/stress.rb",
"chars": 455,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'polyphony'\nrequire 'fileutils'\n\nFileUtils.cd(__dir__)\n\nd"
},
{
"path": "df/ws_page.html",
"chars": 1084,
"preview": "<!doctype html>\n<html lang=\"en\">\n<head>\n <title>Websocket Client</title>\n</head>\n<body>\n <h1>WebSocket Client</h1>\n <"
},
{
"path": "docs/README.md",
"chars": 2956,
"preview": "# Polyphony - Easy Concurrency for Ruby\n\n> Polyphony \\| pəˈlɪf\\(ə\\)ni \\|\n> 1. _Music_ the style of simultaneously combin"
},
{
"path": "examples/cuba.ru",
"chars": 377,
"preview": "# frozen_string_literal: true\n\nrequire 'cuba'\nrequire 'cuba/safe'\nrequire 'delegate' # See https://github.com/rack/rack/"
},
{
"path": "examples/full_service.rb",
"chars": 375,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\n::Exception.__disable_sanitized_backtrace__ = tru"
},
{
"path": "examples/hanami-api.ru",
"chars": 255,
"preview": "# frozen_string_literal: true\n\nrequire 'hanami/api'\n\nclass ExampleApi < Hanami::API\n get '/hello' do\n 'Hello world!'"
},
{
"path": "examples/hello.rb",
"chars": 76,
"preview": "# frozen_string_literal: true\n\nrun { |req|\n req.respond('Hello, world!')\n}\n"
},
{
"path": "examples/hello.ru",
"chars": 129,
"preview": "# frozen_string_literal: true\n\nrun lambda { |env|\n [\n 200,\n {\"Content-Type\" => \"text/plain\"},\n [\"Hello, world!"
},
{
"path": "examples/http1_parser.rb",
"chars": 1227,
"preview": "# frozen_string_literal: true\n\nrequire 'polyphony'\nrequire_relative '../lib/tipi_ext'\n\ni, o = IO.pipe\n\nmodule ::Kernel\n "
},
{
"path": "examples/http_request_ws_server.rb",
"chars": 651,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'tipi/websocket'\n\ndef ws_handler(conn)\n t"
},
{
"path": "examples/http_server.js",
"chars": 671,
"preview": "// For the sake of comparing performance, here's a node.js-based HTTP server\n// doing roughly the same thing as http_ser"
},
{
"path": "examples/http_server.rb",
"chars": 801,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\nopts = {\n reuse_addr: true,\n dont_linger: true"
},
{
"path": "examples/http_server_forked.rb",
"chars": 595,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\n::Exception.__disable_sanitized_backtrace__ = tru"
},
{
"path": "examples/http_server_form.rb",
"chars": 402,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\nopts = {\n reuse_addr: true,\n dont_linger: true"
},
{
"path": "examples/http_server_graceful.rb",
"chars": 414,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'polyphony'\nrequire 'tipi'\n\nopts = {\n reuse_addr: true,"
},
{
"path": "examples/http_server_routes.rb",
"chars": 494,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\nopts = {\n reuse_addr: true,\n dont_linger: true"
},
{
"path": "examples/http_server_simple.rb",
"chars": 205,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\nputs \"pid: #{Process.pid}\"\nputs 'Listening on por"
},
{
"path": "examples/http_server_static.rb",
"chars": 464,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'fileutils'\n\nopts = {\n reuse_addr: true,"
},
{
"path": "examples/http_server_throttled.rb",
"chars": 362,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\n$throttler = Polyphony::Throttler.new(1000)\nopts "
},
{
"path": "examples/http_server_throttled_accept.rb",
"chars": 477,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\n::Exception.__disable_sanitized_backtrace__ = tru"
},
{
"path": "examples/http_server_timeout.rb",
"chars": 562,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\nopts = {\n reuse_addr: true,\n dont_linger: true"
},
{
"path": "examples/http_unix_socket_server.rb",
"chars": 314,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\npath = '/tmp/tipi.sock'\n\nputs \"pid: #{Process.pid"
},
{
"path": "examples/http_ws_server.rb",
"chars": 648,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'tipi/websocket'\n\ndef ws_handler(conn)\n t"
},
{
"path": "examples/https_server.rb",
"chars": 1439,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'localhost/authority'\n\n::Exception.__disab"
},
{
"path": "examples/https_server_forked.rb",
"chars": 643,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'localhost/authority'\n\n::Exception.__disab"
},
{
"path": "examples/https_wss_server.rb",
"chars": 819,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'tipi/websocket'\nrequire 'localhost/author"
},
{
"path": "examples/rack_server.rb",
"chars": 454,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\napp_path = ARGV.first || File.expand_path('./conf"
},
{
"path": "examples/rack_server_forked.rb",
"chars": 625,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\napp_path = ARGV.first || File.expand_path('./conf"
},
{
"path": "examples/rack_server_https.rb",
"chars": 446,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'localhost/authority'\n\napp_path = ARGV.fir"
},
{
"path": "examples/rack_server_https_forked.rb",
"chars": 641,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'localhost/authority'\n\napp_path = ARGV.fir"
},
{
"path": "examples/routing_server.rb",
"chars": 532,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\n\nopts = {\n reuse_addr: true,\n dont_linger: true"
},
{
"path": "examples/servername_cb.rb",
"chars": 905,
"preview": "# frozen_string_literal: true\n\nrequire 'openssl'\nrequire 'fiber'\n\nctx = OpenSSL::SSL::SSLContext.new\n\nf = Fiber.new { |p"
},
{
"path": "examples/source.rb",
"chars": 72,
"preview": "# frozen_string_literal: true\n\nrun { |req|\n req.serve_file(__FILE__)\n}\n"
},
{
"path": "examples/streaming.rb",
"chars": 196,
"preview": "# frozen_string_literal: true\n\nrun { |req|\n req.send_headers('Content-Type' => 'text/event-stream')\n 10.times { |i|\n "
},
{
"path": "examples/websocket_client.rb",
"chars": 1394,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'polyphony'\nrequire 'websocket'\n\n::Exception.__disable_sa"
},
{
"path": "examples/websocket_demo.rb",
"chars": 1802,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'tipi/websocket'\n\nclass WebsocketClient\n "
},
{
"path": "examples/websocket_secure_server.rb",
"chars": 562,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'localhost/authority'\n\ndef ws_handler(conn"
},
{
"path": "examples/websocket_server.rb",
"chars": 460,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'tipi'\nrequire 'tipi/websocket'\n\ndef ws_handler(conn)\n w"
},
{
"path": "examples/ws_page.html",
"chars": 881,
"preview": "<!doctype html>\n<html lang=\"en\">\n<head>\n <title>Websocket Client</title>\n</head>\n<body>\n <script>\n var connect = fu"
},
{
"path": "examples/wss_page.html",
"chars": 893,
"preview": "<!doctype html>\n<html lang=\"en\">\n<head>\n <title>Websocket Client</title>\n</head>\n<body>\n <script>\n var connect = fu"
},
{
"path": "examples/zlib-bench.rb",
"chars": 416,
"preview": "# frozen_string_literal: true\n\nFILE_SIZES = {\n 's.tmp' => 2**10,\n 'm.tmp' => 2**17,\n 'l.tmp' => 2**20,\n 'xl.tmp' "
},
{
"path": "lib/tipi/acme.rb",
"chars": 8565,
"preview": "# frozen_string_literal: true\n\nrequire 'openssl'\nrequire 'acme-client'\nrequire 'localhost/authority'\n\nmodule Tipi\n modu"
},
{
"path": "lib/tipi/cli.rb",
"chars": 2463,
"preview": "# frozen_string_literal: true\n\nrequire 'tipi'\nrequire 'fileutils'\nrequire 'tipi/supervisor'\nrequire 'optparse'\n\nmodule T"
},
{
"path": "lib/tipi/config_dsl.rb",
"chars": 3431,
"preview": "# frozen_string_literal: true\n\nmodule Tipi\n module Configuration\n class Interpreter\n # make_blank_slate\n\n "
},
{
"path": "lib/tipi/configuration.rb",
"chars": 1408,
"preview": "# frozen_string_literal: true\n\nrequire_relative './handler'\n\nmodule Tipi\n module Configuration\n class << self\n "
},
{
"path": "lib/tipi/controller/bare_polyphony.rb",
"chars": 0,
"preview": ""
},
{
"path": "lib/tipi/controller/bare_stock.rb",
"chars": 123,
"preview": "# frozen_string_literal: true\n\nmodule Tipi\n module Apps\n module Bare\n def start(opts)\n end\n end\n end\ne"
},
{
"path": "lib/tipi/controller/extensions.rb",
"chars": 734,
"preview": "# frozen_string_literal: true\n\nrequire 'tipi'\n\nmodule Kernel\n def run(app = nil, &block)\n Tipi.app = app || block\n "
},
{
"path": "lib/tipi/controller/stock_http1_adapter.rb",
"chars": 217,
"preview": "# frozen_string_literal: true\n\nrequire 'tipi/http1_adapter'\n\nmodule Tipi\n class StockHTTP1Adapter < HTTP1Adapter\n de"
},
{
"path": "lib/tipi/controller/web_polyphony.rb",
"chars": 9965,
"preview": "# frozen_string_literal: true\n\nrequire 'tipi'\nrequire 'localhost/authority'\nrequire_relative './extensions'\n\nmodule Tipi"
},
{
"path": "lib/tipi/controller/web_stock.rb",
"chars": 17647,
"preview": "# frozen_string_literal: true\n\nrequire 'ever'\nrequire 'localhost/authority'\nrequire 'http/parser'\nrequire 'qeweney'\nrequ"
},
{
"path": "lib/tipi/controller.rb",
"chars": 275,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\nrequire 'json'\n\n# get opts from STDIN\nopts = JSON.parse(ARGV[0]) "
},
{
"path": "lib/tipi/digital_fabric/agent.rb",
"chars": 6403,
"preview": "# frozen_string_literal: true\n\nrequire_relative './protocol'\nrequire_relative './request_adapter'\n\nrequire 'msgpack'\nreq"
},
{
"path": "lib/tipi/digital_fabric/agent_proxy.rb",
"chars": 9301,
"preview": "# frozen_string_literal: true\n\nrequire_relative './protocol'\nrequire 'msgpack'\nrequire 'tipi/websocket'\n\nmodule DigitalF"
},
{
"path": "lib/tipi/digital_fabric/executive/index.html",
"chars": 2112,
"preview": "<!doctype html>\n<html lang=\"en\">\n<head>\n <title>Digital Fabric Executive</title>\n <style>\n .hidden { display: none "
},
{
"path": "lib/tipi/digital_fabric/executive.rb",
"chars": 2855,
"preview": "# frozen_string_literal: true\n\nrequire 'tipi/digital_fabric'\nrequire 'json'\n\nmodule DigitalFabric\n # agent for managing"
},
{
"path": "lib/tipi/digital_fabric/protocol.rb",
"chars": 3104,
"preview": "# frozen_string_literal: true\n\nmodule DigitalFabric\n module Protocol\n PING = 'ping'\n SHUTDOWN = 'shutdown'\n UN"
},
{
"path": "lib/tipi/digital_fabric/request_adapter.rb",
"chars": 899,
"preview": "# frozen_string_literal: true\n\nrequire_relative './protocol'\n\nmodule DigitalFabric\n class RequestAdapter\n def initia"
},
{
"path": "lib/tipi/digital_fabric/service.rb",
"chars": 7061,
"preview": "# frozen_string_literal: true\n\nrequire_relative './protocol'\nrequire_relative './agent_proxy'\nrequire 'securerandom'\n\nmo"
},
{
"path": "lib/tipi/digital_fabric.rb",
"chars": 136,
"preview": "module DigitalFabric\nend\n\n::DF = DigitalFabric\n\nrequire_relative 'digital_fabric/service'\nrequire_relative 'digital_fabr"
},
{
"path": "lib/tipi/handler.rb",
"chars": 828,
"preview": "# frozen_string_literal: true\n\nrequire_relative './rack_adapter'\nrequire_relative './http1_adapter'\nrequire_relative './"
},
{
"path": "lib/tipi/http1_adapter.rb",
"chars": 9079,
"preview": "# frozen_string_literal: true\n\nrequire 'h1p'\nrequire 'qeweney/request'\n\nrequire_relative './http2_adapter'\n\nmodule Tipi\n"
},
{
"path": "lib/tipi/http2_adapter.rb",
"chars": 2904,
"preview": "# frozen_string_literal: true\n\nrequire 'http/2'\nrequire_relative './http2_stream'\n\n# patch to fix bug in HTTP2::Stream\nc"
},
{
"path": "lib/tipi/http2_stream.rb",
"chars": 5555,
"preview": "# frozen_string_literal: true\n\nrequire 'http/2'\nrequire 'qeweney/request'\n\nmodule Tipi\n # Manages an HTTP 2 stream\n cl"
},
{
"path": "lib/tipi/rack_adapter.rb",
"chars": 696,
"preview": "# frozen_string_literal: true\n\nrequire 'rack'\n\nmodule Tipi\n module RackAdapter\n class << self\n def run(app)\n "
},
{
"path": "lib/tipi/response_extensions.rb",
"chars": 414,
"preview": "# frozen_string_literal: true\n\nrequire 'qeweney/request'\n\nmodule Tipi\n module ResponseExtensions\n SPLICE_CHUNKS_SIZE"
},
{
"path": "lib/tipi/supervisor.rb",
"chars": 1975,
"preview": "# frozen_string_literal: true\n\nrequire 'polyphony'\nrequire 'json'\n\nmodule Tipi\n module Supervisor\n class << self\n "
},
{
"path": "lib/tipi/version.rb",
"chars": 66,
"preview": "# frozen_string_literal: true\n\nmodule Tipi\n VERSION = '0.56'\nend\n"
},
{
"path": "lib/tipi/websocket.rb",
"chars": 1297,
"preview": "# frozen_string_literal: true\n\nrequire 'digest/sha1'\nrequire 'websocket'\n\nmodule Tipi\n # Websocket connection\n class W"
},
{
"path": "lib/tipi.rb",
"chars": 1754,
"preview": "# frozen_string_literal: true\n\nrequire 'polyphony'\n\nrequire_relative './tipi/http1_adapter'\nrequire_relative './tipi/htt"
},
{
"path": "test/coverage.rb",
"chars": 1092,
"preview": "# frozen_string_literal: true\n\nrequire 'coverage'\nrequire 'simplecov'\n\nclass << SimpleCov::LinesClassifier\n alias_metho"
},
{
"path": "test/eg.rb",
"chars": 620,
"preview": "# frozen_string_literal: true\n\nmodule Kernel\n RE_CONST = /^[A-Z]/.freeze\n RE_ATTR = /^@(.+)$/.freeze\n\n def eg(hash"
},
{
"path": "test/helper.rb",
"chars": 2583,
"preview": "# frozen_string_literal: true\n\nrequire 'bundler/setup'\n\nrequire 'fileutils'\nrequire_relative './eg'\n\nrequire_relative '."
},
{
"path": "test/run.rb",
"chars": 99,
"preview": "# frozen_string_literal: true\n\nDir.glob(\"#{__dir__}/test_*.rb\").each do |path|\n require(path)\nend\n"
},
{
"path": "test/test_http_server.rb",
"chars": 6018,
"preview": "# frozen_string_literal: true\n\nrequire_relative 'helper'\nrequire 'tipi'\n\nclass String\n def crlf_lines\n gsub \"\\n\", \"\\"
},
{
"path": "test/test_request.rb",
"chars": 1976,
"preview": "# frozen_string_literal: true\n\nrequire_relative 'helper'\nrequire 'tipi'\n\nclass String\n def http_lines\n gsub \"\\n\", \"\\"
},
{
"path": "tipi.gemspec",
"chars": 2206,
"preview": "require_relative './lib/tipi/version'\n\nGem::Specification.new do |s|\n s.name = 'tipi'\n s.version = Tipi::VE"
}
]
About this extraction
This page contains the full source code of the digital-fabric/tipi GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 99 files (173.1 KB), approximately 49.2k tokens, and a symbol index with 519 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.