Repository: dabeaz/thredo Branch: master Commit: bd17c885bdad Files: 38 Total size: 76.1 KB Directory structure: gitextract_m8p9z_v2/ ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── examples/ │ ├── echo2.py │ ├── echoss.py │ ├── euro/ │ │ ├── README.txt │ │ ├── ex1.py │ │ ├── ex2.py │ │ ├── ex3.py │ │ ├── ex4.py │ │ ├── ex5.py │ │ ├── ex6.py │ │ ├── ex7.py │ │ └── serv.py │ ├── ex1.py │ ├── ex2.py │ ├── ex3.py │ ├── happy.py │ ├── prod.py │ ├── treq.py │ └── tut.py ├── setup.py ├── tests/ │ ├── test_core.py │ ├── test_io.py │ ├── test_queue.py │ └── test_sync.py └── thredo/ ├── __init__.py ├── core.py ├── io.py ├── magic.py ├── mixin.py ├── queue.py ├── requests.py ├── signal.py ├── socket.py ├── sync.py └── thr.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2018 David Beazley Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MANIFEST.in ================================================ recursive-include examples * recursive-include docs * recursive-include tests * include README.rst include LICENSE ================================================ FILE: README.md ================================================ # thredo ## Thredo is Dead Thread was a research topic related to the mixing of threads and async. I spoke about it at EuroPython 2018 (https://www.youtube.com/watch?v=U66KuyD3T0M). However, due to a lack of time and outside interest, the project has been abandoned. Feel free to look around however. -- Dave ## Introduction Thredo is threads on async. For the brave. Or the foolish. Only time will tell. Note: Thredo requires the most up-to-date version of Curio--meaning the one that is checked out of the Curio GitHub repository at https://github.com/dabeaz/curio. ## High Level Overview (The Big Idea) Consider the following thread program involving a worker, a producer, and queue:: import threading import queue import time def worker(q): while True: item = q.get() if item is None: break print('Got:', item) def main(): q = queue.Queue() t = threading.Thread(target=worker, args=(q,)) t.start() for n in range(10): q.put(n) time.sleep(1) q.put(None) t.join() main() In this code, there are blocking operations such as ``q.get()`` and ``time.sleep()``. This blocking is ultimately handled by the host operating system. Because of that, it is very difficult for Python to do anything related to the actual control or scheduling of threads. Once blocked, a thread stays blocked forever or until some event occurs that causes it to unblock. Thredo re-envisions threads by redirecting all blocking operations to an async library. The code looks mostly the same except that you use the `thredo` module. For example: import thredo def worker(q): while True: item = q.get() if item is None: break print('Got:', item) def main(): q = thredo.Queue() t = thredo.spawn(worker, q) for n in range(10): q.put(n) thredo.sleep(1) q.put(None) t.join() thredo.run(main) The main reason you'd use ``thredo`` however is that it gives you extra features such as thread groups, cancellation, and more. For example, here's a more advanced version of the above code:: import thredo def worker(q): try: while True: item = q.get() print('Got:', item) q.task_done() except thredo.ThreadCancelled: print('Worker cancelled') def main(): q = thredo.Queue() with thredo.ThreadGroup(wait=None) as workers: for n in range(4): workers.spawn(worker, q) for n in range(10): q.put(n) thredo.sleep(1) q.join() thredo.run(main) ## Examples The ``examples`` directory contains more examples of using ``thredo``. The ``examples/euro`` directory contains coding samples from the EuroPython 2018 talk. ## FAQ **Q: Is this going to turn into a full-fledged project?** A: It's too early to say. **Q: Isn't this sort of like using concurrent.futures?** A: No. concurrent.futures provides no mechanism for controlling threads and no mechanism for cancelling threads. Although it might appear like this is so, given that you can seemingly "cancel" a Future, this has no effect on thread execution. Once started, worked submitted to a thread pool in concurrent.futures runs to completion regardless of whether or not the associated Future is cancelled. Cancelling a future really only causes it to be abandoned if it hasn't yet started. If you cancel a thread in thredo, it is cleanly cancelled at the next blocking operation. **Q: Are Thredo Threads real Threads?** A: Yes. All threads are created using ``threading.Thread``. They run concurrently according to the same rules as normal threads and can use all of the objects normally associated with threads (including synchronization primitives in ``threading``). If you want to use things like thread-cancellation however, you need to make sure you use various objects provided in the ``thredo`` module. For example, waiting on a ``thredo.Lock`` can be cancelled whereas waiting on a ``threading.Lock`` can not. ================================================ FILE: examples/echo2.py ================================================ # Simple echo server benchmark from thredo.socket import * import thredo def echo_handler(client, addr): print('Connection from', addr) try: f = client.makefile('rb') g = client.makefile('wb') for line in f: g.write(line) finally: print('Connection closed') f.close() g.close() client.close() def killer(delay, t): thredo.sleep(delay) t.cancel() def echo_server(host, port): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) sock.bind((host,port)) sock.listen(5) while True: client, addr = sock.accept() t = thredo.spawn(echo_handler, client, addr, daemon=True) thredo.spawn(killer, 30, t) thredo.run(echo_server, '', 25000) ================================================ FILE: examples/echoss.py ================================================ # Echo server implemented using socket server and # a ThredoMixIn class. This class replaces the normal # socket with one that can be cancelled. Also uses spawn() # internally to launch threads. from thredo.socket import * import thredo import socketserver import signal class EchoHandler(socketserver.BaseRequestHandler): def handle(self): print("Connection from", self.client_address) try: while True: data = self.request.recv(100000) if not data: break self.request.sendall(data) except thredo.ThreadCancelled: print('Handler Cancelled') print('Connection closed') class EchoStreamHandler(socketserver.StreamRequestHandler): def handle(self): print("Stream Connection from", self.client_address) try: for line in self.rfile: self.wfile.write(line) except thredo.ThreadCancelled: print('Stream Handler Cancelled') print('Stream Connection closed') class ThredoTCPServer(thredo.ThredoMixIn, socketserver.TCPServer): pass allow_reuse_address = True def main(): # serv = ThredoTCPServer(('', 25000), EchoHandler) serv = ThredoTCPServer(('', 25000), EchoStreamHandler) serv.allow_reuse_address = True t = thredo.spawn(serv.serve_forever) thredo.SignalEvent(signal.SIGINT).wait() print('Cancelling') t.cancel() thredo.run(main) ================================================ FILE: examples/euro/README.txt ================================================ Example programs written during EuroPython keynote. July 23, 2018. ================================================ FILE: examples/euro/ex1.py ================================================ import thredo def func(x, y): return x + y def main(): t = thredo.spawn(func, 2, '3') try: print('Result:', t.join()) except thredo.ThreadError as e: print('Failed:', repr(e.__cause__)) thredo.run(main) ================================================ FILE: examples/euro/ex2.py ================================================ # Example 2: Real threads import thredo def func(n, label): total = 0 while n > 0: total += n n -= 1 if n % 10_000_000 == 0: print(label, n) return total def main(): t1 = thredo.spawn(func, 60_000_000, 'thread-1') t2 = thredo.spawn(func, 40_000_000, 'thread-2') print(t1.join()) print(t2.join()) thredo.run(main) ================================================ FILE: examples/euro/ex3.py ================================================ # example 3: Cancellation import thredo def func(): print('Yawn') try: thredo.sleep(10) print('Awake') except thredo.ThreadCancelled: print('Cancelled') def main(): t = thredo.spawn(func) thredo.sleep(2) t.cancel() #t.wait() thredo.run(main) ================================================ FILE: examples/euro/ex4.py ================================================ # Example : Cancellation with locks import thredo def func(lck, label): print(label, 'starting') with lck: print(label, 'working') thredo.sleep(5) print(label, 'done') def main(): lck = thredo.Semaphore() t1 = thredo.spawn(func, lck, 'thread-1') t2 = thredo.spawn(func, lck, 'thread-2') # Case 2: Cancel a thread holding a lock thredo.sleep(2) t1.cancel() t2.join() # t2.join() thredo.run(main) ================================================ FILE: examples/euro/ex5.py ================================================ # ex5.py import thredo import random sticks = [thredo.Lock() for n in range(5)] def philosopher(n): thredo.sleep(random.random()) try: with sticks[n]: thredo.sleep(random.random()) with sticks[(n+1) % 5]: print("eating", n) thredo.sleep(random.random()) except thredo.ThreadCancelled: print(f"But what is death? {n}") def main(): phils = [ thredo.spawn(philosopher, n) for n in range(5) ] try: with thredo.timeout_after(10): for p in phils: p.wait() except thredo.ThreadTimeout: phils[random.randint(0,4)].cancel() for p in phils: p.wait() thredo.run(main) ================================================ FILE: examples/euro/ex6.py ================================================ # Example 6 - Queues import thredo def worker(q, label): try: while True: item = q.get() print(label, item) q.task_done() except thredo.ThreadCancelled: print("Cancelled", label) def spin(n): while n > 0: n -= 1 if n % 10_000_000 == 0: print('spin', n) def main(): # thredo.spawn(spin, 60_000_000) q = thredo.Queue() with thredo.ThreadGroup(wait=None) as g: g.spawn(spin,100_000_000) for n in range(4): g.spawn(worker, q, f'Worker-{n}') for n in range(10): q.put(n) thredo.sleep(1) q.join() print('Done') thredo.run(main) ================================================ FILE: examples/euro/ex7.py ================================================ # Example : Die Requests # # Note: This requests the serv.py program to be running in the background import thredo with thredo.more_magic(): import requests def func(): try: r = requests.get('http://localhost:8000') return r.text except thredo.ThreadCancelled: print("Cancelled") def main(): with thredo.ThreadGroup(wait=any) as g: for n in range(4): g.spawn(func) print('Result:', g.completed.result) thredo.run(main) ================================================ FILE: examples/euro/serv.py ================================================ from socket import * import random import time def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) sock.bind(address) sock.listen(5) while True: client, addr = sock.accept() handler(client, addr) def handler(client, address): time.sleep(random.randint(5,20)) client.sendall(b'HTTP/1.0 200 OK\r\n\r\nDie Threads!\n') client.close() if __name__ == '__main__': server(('',8000)) ================================================ FILE: examples/ex1.py ================================================ # Example 1 # # Launching a thread and collecting its result import thredo def func(x, y): return x + y def main(): t = thredo.spawn(func, 2, 3) result = t.join() print('Result:', result) thredo.run(main) ================================================ FILE: examples/ex2.py ================================================ # Example 2 # # Launching multiple CPU-bound threads and collecting their results import thredo def sum_to_n(n, label): total = 0 while n > 0: total += n n -= 1 if n % 10000000 == 0: print(label, n) return total def main(): t1 = thredo.spawn(sum_to_n, 50_000_000, 'thread 1') t2 = thredo.spawn(sum_to_n, 30_000_000, 'thread 2') print('Result 1:', t1.join()) print('Result 2:', t2.join()) thredo.run(main) ================================================ FILE: examples/ex3.py ================================================ # Example 3 # # Cancelling a thread import thredo def hello(sec): print("Yawn") try: thredo.sleep(sec) print("Awake") except thredo.ThreadCancelled: print("Cancelled!") def main(): t = thredo.spawn(hello, 100) thredo.sleep(5) t.cancel() print("Goodbye") thredo.run(main) ================================================ FILE: examples/happy.py ================================================ # happy.py # An implementation of RFC 6555 (Happy Eyeballs) from thredo import socket, ThreadGroup, spawn, ignore_after, run import itertools def open_tcp_stream(hostname, port, happy_eyeballs_delay=0.3): # Get all of the possible targets for a given host/port targets = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) if not targets: raise OSError(f'nothing known about {hostname}:{port}') # Group the targets into different address families families = { af: list(group) for af, group in itertools.groupby(targets, key=lambda t: t[0]) } # Arrange the targets to interleave address families. targets = itertools.chain(*zip(*families.values())) # List of socket-related errors (if any) errors = [] def try_connect(sock, addr): try: sock.connect(addr) print("Got:", sock) return sock except OSError as e: errors.append(e) sock.close() return None def connector(group): for *sockargs, _, addr in targets: sock = socket.socket(*sockargs) task = group.spawn(try_connect, sock, addr) with ignore_after(happy_eyeballs_delay): task.wait() print("here!") with ThreadGroup(wait=any) as g: g.spawn(connector, g) if g.completed: return g.completed.result else: raise OSError(errors) def main(): result = open_tcp_stream('www.python.org', 80) print(result) if __name__ == '__main__': run(main) ================================================ FILE: examples/prod.py ================================================ import thredo def consumer(q): while True: item = q.get() if item is None: break print("Child:", item) thredo.sleep(0.01) def producer(q, count): for n in range(count): q.put(n) print("Produced:", n) q.put(None) def main(): q = thredo.Queue(maxsize=1) t1 = thredo.spawn(consumer, q) t2 = thredo.spawn(producer, q, 1000) t1.join() t2.join() thredo.run(main) ================================================ FILE: examples/treq.py ================================================ import thredo import thredo.requests def get(session, url): try: return session.get(url) except thredo.ThreadCancelled as e: print("Cancelled", url) def main(): s = thredo.requests.get_session() with thredo.ThreadGroup(wait=any) as g: g.spawn(get, s, 'http://www.dabeaz.com/cgi-bin/saas.py?s=5') g.spawn(get, s, 'http://www.dabeaz.com/cgi-bin/saas.py?s=10') g.spawn(get, s, 'http://www.dabeaz.com/cgi-bin/fib.py?n=10') print(g.completed.result.text) if __name__ == '__main__': thredo.run(main) ================================================ FILE: examples/tut.py ================================================ import thredo import signal def countdown(n): while n > 0: print('T-minus', n) thredo.sleep(1) n -= 1 def friend(name): print('Hi, my name is', name) print('Playing Minecraft') try: thredo.sleep(1000) except thredo.CancelledError: print(name, 'going home') raise start_evt = thredo.Event() def kid(): while True: try: print('Can I play?') thredo.timeout_after(1, start_evt.wait) break except thredo.ThreadTimeout: print('Wha!?!') print('Building the Millenium Falcon in Minecraft') with thredo.ThreadGroup() as f: f.spawn(friend, 'Max') f.spawn(friend, 'Lillian') f.spawn(friend, 'Thomas') try: thredo.sleep(1000) except thredo.CancelledError: print('Fine. Saving my work.') raise def parent(): goodbye = thredo.SignalEvent(signal.SIGINT, signal.SIGTERM) kid_task = thredo.spawn(kid) thredo.sleep(5) print("Yes, go play") start_evt.set() goodbye.wait() del goodbye print("Let's go") count_task = thredo.spawn(countdown, 10) count_task.join() print("We're leaving!") try: thredo.timeout_after(10, kid_task.join) except thredo.ThreadTimeout: kid_task.cancel() thredo.sleep(3) print('Parent Leaving') if __name__ == '__main__': thredo.run(parent) ================================================ FILE: setup.py ================================================ try: from setuptools import setup except ImportError: from distutils.core import setup tests_require = ['pytest'] long_description = """ Thredo is threads. """ setup(name="thredo", description="Thredo - Threads", long_description=long_description, license="MIT", version="0.0", author="David Beazley", author_email="dave@dabeaz.com", maintainer="David Beazley", maintainer_email="dave@dabeaz.com", url="https://github.com/dabeaz/thredo", packages=['thredo'], tests_require=tests_require, extras_require={ 'test': tests_require, }, classifiers=[ 'Programming Language :: Python :: 3', ]) ================================================ FILE: tests/test_core.py ================================================ # test_core.py # # Tests for core functions and threads import thredo import time def basic_thread(x, y): return x + y def test_good(): result = thredo.run(basic_thread, 2, 2) assert result == 4 def test_bad(): try: result = thredo.run(basic_thread, 2, '2') assert False except BaseException as e: assert type(e) == TypeError def test_good_spawn(): result = [] def main(): t = thredo.spawn(basic_thread, 2, 2) result.append(t.join()) thredo.run(main) assert result == [4] def test_bad_spawn(): result = [] def main(): t = thredo.spawn(basic_thread, 2, '2') try: t.join() except thredo.ThreadError as e: result.append('error') result.append(type(e.__cause__)) thredo.run(main) assert result == ['error', TypeError] def test_sleep(): result = [] def main(): start = time.time() thredo.sleep(1.0) end = time.time() result.append(end-start) thredo.run(main) assert result[0] > 1.0 def test_sleep_cancel(): result = [] def child(): try: thredo.sleep(10) except thredo.ThreadCancelled: result.append('cancel') def main(): t = thredo.spawn(child) thredo.sleep(0.1) t.cancel() thredo.run(main) assert result == ['cancel'] def test_timeout(): result = [] def main(): try: thredo.timeout_after(0.25, thredo.sleep, 1) except thredo.ThreadTimeout: result.append('timeout') thredo.run(main) assert result == ['timeout'] def test_timeout_context(): result = [] def main(): try: with thredo.timeout_after(0.25): thredo.sleep(1) except thredo.ThreadTimeout: result.append('timeout') thredo.run(main) assert result == ['timeout'] def test_ignore(): def main(): thredo.ignore_after(0.25, thredo.sleep, 1) start = time.time() thredo.run(main) end = time.time() assert (end-start) < 1 def test_ignore_context(): def main(): with thredo.ignore_after(0.25): thredo.sleep(1) start = time.time() thredo.run(main) end = time.time() assert (end-start) < 1 ================================================ FILE: tests/test_io.py ================================================ # test_io.py import thredo from thredo.socket import * def test_read_partial(): results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) client, addr = sock.accept() try: client.send(b'OK') s = client.as_stream() line = s.readline() results.append(line) data = s.read(2) results.append(data) data = s.read(1000) results.append(data) finally: client.close() sock.close() def test_client(address): sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'hello\nworld\n') sock.close() def main(): serv = thredo.spawn(server, ('',25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ b'hello\n', b'wo', b'rld\n'] def test_readall(): results = [] evt = thredo.Event() def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: results.append('handler start') client.send(b'OK') with client.as_stream() as s: data = s.readall() results.append(data) results.append('handler done') finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\n') thredo.sleep(0.1) sock.send(b'Msg2\n') thredo.sleep(0.1) sock.send(b'Msg3\n') thredo.sleep(0.1) sock.close() def main(): serv = thredo.spawn(server, ('',25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\nMsg2\nMsg3\n', 'handler done' ] def test_readall_partial(): results = [] evt = thredo.Event() def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: client.send(b'OK') s = client.as_stream() line = s.readline() results.append(line) data = s.readall() results.append(data) s.close() finally: client.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'hello\nworld\n') sock.close() def main(): serv = thredo.spawn(server, ('',25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [b'hello\n', b'world\n'] def test_readall_timeout(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: results.append('handler start') client.send(b'OK') s = client.as_stream() try: data = thredo.timeout_after(0.5, s.readall) except thredo.ThreadTimeout as e: results.append(e.bytes_read) results.append('handler done') finally: sock.close() client.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\n') thredo.sleep(1) sock.send(b'Msg2\n') sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\n', 'handler done' ] def test_read_exactly(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: results.append('handler start') client.send(b'OK') s = client.as_stream() for n in range(3): results.append(s.read_exactly(5)) results.append('handler done') finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\nMsg2\nMsg3\n') sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\n', b'Msg2\n', b'Msg3\n', 'handler done' ] def test_read_exactly_incomplete(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: client.send(b'OK') s = client.as_stream() try: s.read_exactly(100) except EOFError as e: results.append(e.bytes_read) finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\nMsg2\nMsg3\n') sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results[0] == b'Msg1\nMsg2\nMsg3\n' def test_read_exactly_timeout(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: results.append('handler start') client.send(b'OK') s = client.as_stream() try: data = thredo.timeout_after(0.5, s.read_exactly, 10) results.append(data) except thredo.ThreadTimeout as e: results.append(e.bytes_read) results.append('handler done') finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\n') thredo.sleep(1) sock.send(b'Msg2\n') sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\n', 'handler done' ] def test_readline(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: results.append('handler start') client.send(b'OK') s = client.as_stream() for n in range(3): results.append(s.readline()) results.append('handler done') finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\nMsg2\nMsg3\n') sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\n', b'Msg2\n', b'Msg3\n', 'handler done' ] def test_readlines(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: client.send(b'OK') results.append('handler start') s = client.as_stream() results.extend(s.readlines()) results.append('handler done') finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\nMsg2\nMsg3\n') sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\n', b'Msg2\n', b'Msg3\n', 'handler done' ] def test_readlines_timeout(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: client.send(b'OK') results.append('handler start') s = client.as_stream() try: thredo.timeout_after(0.5, s.readlines) except thredo.ThreadTimeout as e: results.extend(e.lines_read) results.append('handler done') finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\nMsg2\n') thredo.sleep(1) sock.send(b'Msg3\n') sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\n', b'Msg2\n', 'handler done' ] def test_writelines(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: results.append('handler start') client.send(b'OK') s = client.as_stream() results.append(s.readall()) results.append('handler done') finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) s = sock.as_stream() s.writelines([b'Msg1\n', b'Msg2\n', b'Msg3\n']) sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\nMsg2\nMsg3\n', 'handler done' ] def test_writelines_timeout(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: client.send(b'OK') s = client.as_stream() thredo.sleep(1) results.append(s.readall()) finally: client.close() sock.close() def line_generator(): n = 0 while True: yield b'Msg%d\n' % n n += 1 def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) s = sock.as_stream() try: thredo.timeout_after(0.5, s.writelines, line_generator()) except thredo.ThreadTimeout as e: results.append(e.bytes_written) sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results[0] == len(results[1]) def test_write_timeout(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: client.send(b'OK') s = client.as_stream() thredo.sleep(1) results.append(s.readall()) finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) s = sock.as_stream() try: msg = b'x'*10000000 # Must be big enough to fill buffers thredo.timeout_after(0.5, s.write, msg) except thredo.ThreadTimeout as e: results.append(e.bytes_written) sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results[0] == len(results[1]) def test_iterline(): evt = thredo.Event() results = [] def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: results.append('handler start') client.send(b'OK') s = client.as_stream() for line in s: results.append(line) results.append('handler done') finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) sock.recv(8) sock.send(b'Msg1\nMsg2\nMsg3\n') sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) serv.join() client.join() thredo.run(main) assert results == [ 'handler start', b'Msg1\n', b'Msg2\n', b'Msg3\n', 'handler done' ] def test_sendall_cancel(): evt = thredo.Event() start = thredo.Event() results = {} def server(address): sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) sock.bind(address) sock.listen(1) evt.set() client, addr = sock.accept() try: start.wait() nrecv = 0 while True: data = client.recv(1000000) if not data: break nrecv += len(data) results['handler'] = nrecv finally: client.close() sock.close() def test_client(address): evt.wait() sock = socket(AF_INET, SOCK_STREAM) sock.connect(address) try: sock.sendall(b'x'*10000000) except thredo.ThreadCancelled as e: results['sender'] = e.bytes_sent sock.close() def main(): serv = thredo.spawn(server, ('', 25000)) client = thredo.spawn(test_client, ('localhost', 25000)) thredo.sleep(0.1) client.cancel() start.set() serv.join() thredo.run(main) assert results['handler'] == results['sender'] ================================================ FILE: tests/test_queue.py ================================================ # test_queue.py import thredo def test_queue_simple(): results = [] def consumer(q): while True: item = q.get() if item is None: break results.append(item) q.task_done() q.task_done() def producer(q): results.append('start') for n in range(3): q.put(n) q.put(None) q.join() results.append('done') def main(): q = thredo.Queue() t1 = thredo.spawn(consumer, q) t2 = thredo.spawn(producer, q) t1.join() t2.join() thredo.run(main) assert results == ['start', 0, 1, 2, 'done'] def test_queue_get_cancel(): results = [] def consumer(q): while True: try: item = q.get() except thredo.ThreadCancelled: results.append('cancel') raise def main(): q = thredo.Queue() t = thredo.spawn(consumer, q) results.append('start') t.cancel() thredo.run(main) assert results == ['start', 'cancel'] def test_queue_put_cancel(): results = [] def producer(q): while True: try: q.put(True) except thredo.ThreadCancelled: results.append('cancel') raise def main(): q = thredo.Queue(maxsize=1) t = thredo.spawn(producer, q) results.append('start') thredo.sleep(0.1) t.cancel() thredo.run(main) assert results == ['start', 'cancel'] def test_queue_join_cancel(): results = [] def producer(q): q.put(True) try: q.join() except thredo.ThreadCancelled: results.append('cancel') raise def main(): q = thredo.Queue(maxsize=1) t = thredo.spawn(producer, q) results.append('start') thredo.sleep(0.1) t.cancel() thredo.run(main) assert results == ['start', 'cancel'] ================================================ FILE: tests/test_sync.py ================================================ import thredo def test_event_wait(): evt = thredo.Event() result = [] def waiter(): evt.wait() result.append('waiter') def main(): t = thredo.spawn(waiter) result.append('start') evt.set() t.join() thredo.run(main) assert result == ['start', 'waiter'] evt.clear() assert not evt.is_set() def test_event_wait_cancel(): evt = thredo.Event() result = [] def waiter(): try: evt.wait() result.append('waiter') except thredo.ThreadCancelled: result.append('cancel') def main(): t = thredo.spawn(waiter) result.append('start') t.cancel() thredo.run(main) assert result == ['start', 'cancel'] def test_lock(): lock = thredo.Lock() result = [] def child(): with lock: result.append('child') def main(): lock.acquire() if lock.locked(): result.append('locked') try: t = thredo.spawn(child) result.append('parent') finally: lock.release() t.join() thredo.run(main) assert result == ['locked', 'parent', 'child'] def test_lock_cancel(): lock = thredo.Lock() result = [] def child(): try: with lock: result.append('child') except thredo.ThreadCancelled: result.append('cancel') def main(): lock.acquire() try: t = thredo.spawn(child) result.append('parent') t.cancel() finally: lock.release() thredo.run(main) assert result == ['parent', 'cancel'] def test_lock_race(): lock = thredo.Lock() evt = thredo.Event() n = 0 def incr(count): nonlocal n evt.wait() while count > 0: with lock: n += 1 count -=1 def decr(count): nonlocal n evt.wait() while count > 0: with lock: n -= 1 count -= 1 def main(): t1 = thredo.spawn(incr, 10000) t2 = thredo.spawn(decr, 10000) evt.set() t1.join() t2.join() thredo.run(main) assert n == 0 def test_semaphore(): lock = thredo.Semaphore() result = [] def child(): with lock: result.append('child') def main(): lock.acquire() result.append(lock.value) try: t = thredo.spawn(child) result.append('parent') finally: lock.release() t.join() thredo.run(main) assert result == [0, 'parent', 'child'] def test_semaphore_cancel(): lock = thredo.Semaphore() result = [] def child(): try: with lock: result.append('child') except thredo.ThreadCancelled: result.append('cancel') def main(): lock.acquire() try: t = thredo.spawn(child) result.append('parent') t.cancel() finally: lock.release() thredo.run(main) assert result == ['parent', 'cancel'] def test_semaphore_race(): lock = thredo.Semaphore() evt = thredo.Event() n = 0 def incr(count): nonlocal n evt.wait() while count > 0: with lock: n += 1 count -=1 def decr(count): nonlocal n evt.wait() while count > 0: with lock: n -= 1 count -= 1 def main(): t1 = thredo.spawn(incr, 10000) t2 = thredo.spawn(decr, 10000) evt.set() t1.join() t2.join() thredo.run(main) assert n == 0 def test_rlock(): lock = thredo.RLock() result = [] def child(): with lock: result.append('child') def main(): lock.acquire() if lock.locked(): result.append('locked') try: t = thredo.spawn(child) result.append('parent') finally: lock.release() t.join() thredo.run(main) assert result == ['locked', 'parent', 'child'] def test_rlock_cancel(): lock = thredo.RLock() result = [] def child(): try: with lock: result.append('child') except thredo.ThreadCancelled: result.append('cancel') def main(): lock.acquire() try: t = thredo.spawn(child) result.append('parent') t.cancel() finally: lock.release() thredo.run(main) assert result == ['parent', 'cancel'] def test_rlock_race(): lock = thredo.RLock() evt = thredo.Event() n = 0 def incr(count): nonlocal n evt.wait() while count > 0: with lock: n += 1 count -=1 def decr(count): nonlocal n evt.wait() while count > 0: with lock: n -= 1 count -= 1 def main(): t1 = thredo.spawn(incr, 10000) t2 = thredo.spawn(decr, 10000) evt.set() t1.join() t2.join() thredo.run(main) assert n == 0 def test_condition(): lock = thredo.Condition() result = [] def child(): with lock: result.append('child') def main(): lock.acquire() if lock.locked(): result.append('locked') try: t = thredo.spawn(child) result.append('parent') finally: lock.release() t.join() thredo.run(main) assert result == ['locked', 'parent', 'child'] def test_condition_cancel(): lock = thredo.Condition() result = [] def child(): try: with lock: result.append('child') except thredo.ThreadCancelled: result.append('cancel') def main(): lock.acquire() try: t = thredo.spawn(child) result.append('parent') t.cancel() finally: lock.release() thredo.run(main) assert result == ['parent', 'cancel'] def test_condition_race(): lock = thredo.Condition() evt = thredo.Event() n = 0 def incr(count): nonlocal n evt.wait() while count > 0: with lock: n += 1 count -=1 def decr(count): nonlocal n evt.wait() while count > 0: with lock: n -= 1 count -= 1 def main(): t1 = thredo.spawn(incr, 10000) t2 = thredo.spawn(decr, 10000) evt.set() t1.join() t2.join() thredo.run(main) assert n == 0 def test_condition_wait_notify(): n = 0 lock = thredo.Condition(thredo.Lock()) result = [] def waiter(): current = n while True: with lock: while current == n: lock.wait() result.append(('consume', n)) current = n if n >= 5: break def producer(): nonlocal n while n < 5: thredo.sleep(0.1) with lock: n += 1 result.append(('produce', n)) lock.notify() def main(): t1 = thredo.spawn(waiter) t2 = thredo.spawn(producer) t1.join() t2.join() thredo.run(main) assert result == [('produce',1), ('consume',1), ('produce',2), ('consume',2), ('produce',3), ('consume',3), ('produce',4), ('consume',4), ('produce',5), ('consume',5)] ================================================ FILE: thredo/__init__.py ================================================ # __init__.py from .core import * from .sync import * from .signal import * from .queue import * from .mixin import * from .magic import more_magic ================================================ FILE: thredo/core.py ================================================ # core.py __all__ = ['run', 'sleep', 'spawn', 'timeout_after', 'ignore_after', 'ThreadTimeout', 'ThreadCancelled', 'CancelledError', 'ThreadError', 'ThreadGroup'] import curio from .thr import TAWAIT as AWAIT from . import thr class Thread: def __init__(self, atask): self.atask = atask def cancel(self): AWAIT(self.atask.cancel) def join(self): return AWAIT(self.atask.join) def wait(self): return AWAIT(self.atask.wait) class ThreadGroup: def __init__(self, *args, **kwargs): self._tg = curio.TaskGroup(*args, **kwargs) def spawn(self, callable, *args, daemon=False): t = AWAIT(curio.spawn_thread, callable, *args, daemon=daemon) AWAIT(self._tg.add_task, t) return Thread(t) def cancel(self, *args, **kwargs): AWAIT(self._tg.cancel, *args, **kwargs) def cancel_remaining(self, *args, **kwargs): AWAIT(self._tg.cancel_remaining, *args, **kwargs) def join(self, *args, **kwargs): return AWAIT(self._tg.cancel, *args, **kwargs) def next_result(self, *args, **kwargs): return AWAIT(self._tg.next_result, *args, **kwargs) def next_done(self, *args, **kwargs): return AWAIT(self._tg.next_done, *args, **kwargs) def __enter__(self): self._tg.__enter__() return self def __exit__(self, *args): return self._tg.__exit__(*args) @property def completed(self): return self._tg.completed def run(callable, *args): async def _runner(): t = await curio.spawn(thr.thread_handler) try: async with curio.spawn_thread(): return callable(*args) finally: await t.cancel() return curio.run(_runner) def enable(): thr.enable_async() def sleep(seconds): return AWAIT(curio.sleep, seconds) def spawn(callable, *args, daemon=False): atask = AWAIT(curio.spawn_thread, callable, *args, daemon=daemon) return Thread(atask) def timeout_after(delay, callable=None, *args): thr.enable_async() if callable: with curio.timeout_after(delay): return callable(*args) else: return curio.timeout_after(delay) def ignore_after(delay, callable=None, *args): thr.enable_async() if callable: with curio.ignore_after(delay): return callable(*args) else: return curio.ignore_after(delay) ThreadTimeout = curio.TaskTimeout ThreadCancelled = curio.TaskCancelled CancelledError = curio.CancelledError ThreadError = curio.TaskError ================================================ FILE: thredo/io.py ================================================ # io.py __all__ = ['Socket'] # -- Standard Library from socket import SOL_SOCKET, SO_ERROR from contextlib import contextmanager import io import os # -- Curio from curio.traps import _read_wait, _write_wait from curio.io import _Fd, WantRead, WantWrite import curio.errors # -- Thredo from .thr import TAWAIT as AWAIT # This socket class mirrors the functionality in the Curio socket # class. An important facet of the design is that it still relies # upon non-blocking I/O and eager evaluation. Curio only enters # the picture if operations actually block. class Socket(object): ''' Non-blocking wrapper around a socket object. ''' def __init__(self, sock): self._socket = sock self._socket.setblocking(False) self._fileno = _Fd(sock.fileno()) # Commonly used bound methods self._socket_send = sock.send self._socket_recv = sock.recv def __repr__(self): return '' % (self._socket) def __getattr__(self, name): return getattr(self._socket, name) def fileno(self): return int(self._fileno) def settimeout(self, seconds): if seconds: raise RuntimeError('Use timeout_after() to set a timeout') def gettimeout(self): return None def dup(self): return type(self)(self._socket.dup()) def makefile(self, mode, buffering=0, *, encoding=None, errors=None, newline=None): if 'b' not in mode: raise RuntimeError('File can only be created in binary mode') f = self._socket.makefile(mode, buffering=buffering) return FileStream(f) def as_stream(self): return SocketStream(self._socket) def recv(self, maxsize, flags=0): while True: try: return self._socket_recv(maxsize, flags) except WantRead: AWAIT(_read_wait, self._fileno) except WantWrite: AWAIT(_write_wait, self._fileno) def recv_into(self, buffer, nbytes=0, flags=0): while True: try: return self._socket.recv_into(buffer, nbytes, flags) except WantRead: AWAIT(_read_wait, self._fileno) except WantWrite: AWAIT(_write_wait, self._fileno) def send(self, data, flags=0): while True: try: return self._socket_send(data, flags) except WantWrite: AWAIT(_write_wait, self._fileno) except WantRead: AWAIT(_read_wait, self._fileno) def sendall(self, data, flags=0): with memoryview(data).cast('B') as buffer: total_sent = 0 try: while buffer: try: nsent = self._socket_send(buffer, flags) total_sent += nsent buffer = buffer[nsent:] except WantWrite: AWAIT(_write_wait, self._fileno) except WantRead: AWAIT(_read_wait, self._fileno) except curio.errors.CancelledError as e: e.bytes_sent = total_sent raise def accept(self): while True: try: client, addr = self._socket.accept() client = Socket(client) client.__class__ = type(self) return client, addr except WantRead: AWAIT(_read_wait, self._fileno) def connect_ex(self, address): try: self.connect(address) return 0 except OSError as e: return e.errno def connect(self, address): try: result = self._socket.connect(address) if getattr(self, 'do_handshake_on_connect', False): self.do_handshake() return result except WantWrite: AWAIT(_write_wait, self._fileno) err = self._socket.getsockopt(SOL_SOCKET, SO_ERROR) if err != 0: raise OSError(err, 'Connect call failed %s' % (address,)) if getattr(self, 'do_handshake_on_connect', False): self.do_handshake() def recvfrom(self, buffersize, flags=0): while True: try: return self._socket.recvfrom(buffersize, flags) except WantRead: AWAIT(_read_wait, self._fileno) except WantWrite: AWAIT(_write_wait, self._fileno) def recvfrom_into(self, buffer, bytes=0, flags=0): while True: try: return self._socket.recvfrom_into(buffer, bytes, flags) except WantRead: AWAIT(_read_wait, self._fileno) except WantWrite: AWAIT(_write_wait, self._fileno) def sendto(self, bytes, flags_or_address, address=None): if address: flags = flags_or_address else: address = flags_or_address flags = 0 while True: try: return self._socket.sendto(bytes, flags, address) except WantWrite: AWAIT(_write_wait, self._fileno) except WantRead: AWAIT(_read_wait, self._fileno) def recvmsg(self, bufsize, ancbufsize=0, flags=0): while True: try: return self._socket.recvmsg(bufsize, ancbufsize, flags) except WantRead: AWAIT(_read_wait, self._fileno) def recvmsg_into(self, buffers, ancbufsize=0, flags=0): while True: try: return self._socket.recvmsg_into(buffers, ancbufsize, flags) except WantRead: AWAIT(_read_wait, self._fileno) def sendmsg(self, buffers, ancdata=(), flags=0, address=None): while True: try: return self._socket.sendmsg(buffers, ancdata, flags, address) except WantRead: AWAIT(_write_wait, self._fileno) # Special functions for SSL def do_handshake(self): while True: try: return self._socket.do_handshake() except WantRead: AWAIT(_read_wait, self._fileno) except WantWrite: AWAIT(_write_wait, self._fileno) def close(self): if self._socket: self._socket.close() self._socket = None self._fileno = -1 def shutdown(self, how): if self._socket: self._socket.shutdown(how) def __enter__(self): self._socket.__enter__() return self def __exit__(self, *args): self._socket.__exit__(*args) MAX_READ = 65536 class StreamBase(object): ''' Base class for file-like objects. ''' def __init__(self, fileobj): self._file = fileobj self._fileno = _Fd(fileobj.fileno()) self._buffer = bytearray() def __repr__(self): return '' % (type(self).__name__, self._file) def fileno(self): return int(self._fileno) # ---- Methods that must be implemented in child classes def _read(self, maxbytes=-1): raise NotImplementedError() def write(self, data): raise NotImplementedError() # ---- General I/O methods for streams def read(self, maxbytes=-1): buf = self._buffer if buf: if maxbytes < 0 or len(buf) <= maxbytes: data = bytes(buf) buf.clear() else: data = bytes(buf[:maxbytes]) del buf[:maxbytes] else: data = self._read(maxbytes) return data def readall(self): chunks = [] maxread = 65536 if self._buffer: chunks.append(bytes(self._buffer)) self._buffer.clear() while True: try: chunk = self.read(maxread) except curio.errors.CancelledError as e: e.bytes_read = b''.join(chunks) raise if not chunk: return b''.join(chunks) chunks.append(chunk) if len(chunk) == maxread: maxread *= 2 def read_exactly(self, nbytes): chunks = [] while nbytes > 0: try: chunk = self.read(nbytes) except curio.errors.CancelledError as e: e.bytes_read = b''.join(chunks) raise if not chunk: e = EOFError('Unexpected end of data') e.bytes_read = b''.join(chunks) raise e chunks.append(chunk) nbytes -= len(chunk) return b''.join(chunks) def readinto(self, memory): with memoryview(memory).cast('B') as view: remaining = len(view) total_read = 0 # It's possible that there is data already buffered on this stream. # If so, we have to copy into the memory buffer first. buffered = len(self._buffer) tocopy = remaining if (remaining < buffered) else buffered if tocopy: view[:tocopy] = self._buffer[:tocopy] del self._buffer[:tocopy] remaining -= tocopy total_read += tocopy # To emulate behavior of synchronous readinto(), we read all available # bytes up to the buffer size. while remaining > 0: try: nrecv = self._readinto_impl(view[total_read:total_read+remaining]) # On proper file objects, None might be returned to indicate blocking if nrecv is None: AWAIT(_read_wait, self._fileno) elif nrecv == 0: break else: total_read += nrecv remaining -= nrecv except WantRead: AWAIT(_read_wait, self._fileno) except WantWrite: AWAIT(_write_wait, self._fileno) return total_read def readline(self, maxlen=None): while True: nl_index = self._buffer.find(b'\n') if nl_index >= 0: resp = bytes(self._buffer[:nl_index + 1]) del self._buffer[:nl_index + 1] return resp data = self._read(MAX_READ) if data == b'': resp = bytes(self._buffer) self._buffer.clear() return resp self._buffer.extend(data) def readlines(self): lines = [] try: for line in self: lines.append(line) return lines except curio.errors.CancelledError as e: e.lines_read = lines raise def writelines(self, lines): nwritten = 0 for line in lines: try: self.write(line) nwritten += len(line) except curio.errors.CancelledError as e: e.bytes_written += nwritten raise def flush(self): pass def close(self): self.flush() if self._file: self._file.close() self._file = None self._fileno = -1 def __iter__(self): return self def __next__(self): line = self.readline() if line: return line else: raise StopIteration def __enter__(self): return self def __exit__(self, *args): return self.close() class FileStream(StreamBase): ''' Wrapper around a file-like object. File is put into non-blocking mode. The underlying file must be in binary mode. ''' def __init__(self, fileobj): assert not isinstance(fileobj, io.TextIOBase), 'Only binary mode files allowed' super().__init__(fileobj) os.set_blocking(int(self._fileno), False) # Common bound methods self._file_read = fileobj.read self._readinto_impl = getattr(fileobj, 'readinto', None) self._file_write = fileobj.write def _read(self, maxbytes=-1): while True: # In non-blocking mode, a file-like object might return None if no data is # available. Alternatively, we'll catch the usual blocking exceptions just to be safe try: data = self._file_read(maxbytes) if data is None: AWAIT(_read_wait, self._fileno) else: return data except WantRead: AWAIT(_read_wait, self._fileno) except WantWrite: AWAIT(_write_wait, self._fileno) def write(self, data): nwritten = 0 view = memoryview(data).cast('B') try: while view: try: nbytes = self._file_write(view) if nbytes is None: raise BlockingIOError() nwritten += nbytes view = view[nbytes:] except WantWrite as e: if hasattr(e, 'characters_written'): nwritten += e.characters_written view = view[e.characters_written:] AWAIT(_write_wait, self._fileno) except WantRead: AWAIT(_read_wait, self._fileno) return nwritten except curio.errors.CancelledError as e: e.bytes_written = nwritten raise def flush(self): if not self._file: return while True: try: return self._file.flush() except WantWrite: AWAIT(_write_wait, self._fileno) except WantRead: AWAIT(_read_wait, self._fileno) class SocketStream(StreamBase): ''' Stream wrapper for a socket. ''' def __init__(self, sock): super().__init__(sock) sock.setblocking(False) # Common bound methods self._socket_recv = sock.recv self._readinto_impl = sock.recv_into self._socket_send = sock.send def _read(self, maxbytes=-1): while True: try: data = self._socket_recv(maxbytes if maxbytes > 0 else MAX_READ) return data except WantRead: AWAIT(_read_wait, self._fileno) except WantWrite: AWAIT(_write_wait, self._fileno) def write(self, data): nwritten = 0 view = memoryview(data).cast('B') try: while view: try: nbytes = self._socket_send(view) nwritten += nbytes view = view[nbytes:] except WantWrite: AWAIT(_write_wait, self._fileno) except WantRead: AWAIT(_read_wait, self._fileno) return nwritten except curio.errors.CancelledError as e: e.bytes_written = nwritten raise ================================================ FILE: thredo/magic.py ================================================ # magic.py from contextlib import contextmanager import sys from . import socket __all__ = ['more_magic'] @contextmanager def more_magic(): sockmod = sys.modules['socket'] sys.modules['socket'] = socket try: yield finally: sys.modules['socket'] = sockmod ================================================ FILE: thredo/mixin.py ================================================ # mixin.py __all__ = ['ThredoMixIn'] from . import io from . import core class ThredoMixIn: ''' Mixin class that can be used to make standard socketserver objects to use thredo ''' _threads = None def server_activate(self): super().server_activate() self.socket = io.Socket(self.socket) def serve_forever(self): while True: self.handle_request() def handle_request(self): try: self._handle_request_noblock() except core.ThreadCancelled: threads = self._threads self._threads = None if threads: for thread in threads: thread.cancel() raise def process_request_thread(self, request, client_address): try: self.finish_request(request, client_address) except Exception: self.handle_error(request, client_address) finally: self.shutdown_request(request) def process_request(self, request, client_address): t = core.spawn(self.process_request_thread, request, client_address) if self._threads is None: self._threads = [] self._threads.append(t) def server_close(self): super().server_close() threads = self._threads self._threads = None if threads: for thread in threads: thread.join() ================================================ FILE: thredo/queue.py ================================================ # queue.py # # A basic queue __all__ = [ 'Queue' ] import curio # -- Thredo from .thr import TAWAIT as AWAIT class Queue(object): def __init__(self, maxsize=0): self._queue = curio.Queue(maxsize) def empty(self): return self._queue.empty() def full(self): return self._queue.full() def get(self): return AWAIT(self._queue.get) def join(self): return AWAIT(self._queue.join) def put(self, item): return AWAIT(self._queue.put, item) def qsize(self): return self._queue.qsize() def task_done(self): return AWAIT(self._queue.task_done) def __iter__(self): return self def __next__(self): return self.get() ================================================ FILE: thredo/requests.py ================================================ # requests.py # # Session adapter that allows requests to use thredo socket objects. # This is a bit of plumbing, but it's a clean interface that doesn't # require any monkeypatching or other low-level magic __all__ = ['get_session'] # -- Thredo from . import socket # -- Requests/third party import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3 import PoolManager, HTTPConnectionPool from requests.packages.urllib3 import HTTPSConnectionPool from http.client import HTTPConnection, HTTPSConnection class ThredoAdapter(HTTPAdapter): def init_poolmanager(self, connections, maxsize, block): self.poolmanager = ThredoPoolManager(num_pools=connections, maxsize=maxsize, block=block) class ThredoPoolManager(PoolManager): def _new_pool(self, scheme, host, port): # Important! if scheme == 'http': return ThredoHTTPConnectionPool(host, port, **self.connection_pool_kw) if scheme == 'https': return ThredoHTTPSConnectionPool(host, port, **self.connection_pool_kw) return super(PoolManager, self)._new_pool(self, scheme, host, port) class ThredoHTTPConnectionPool(HTTPConnectionPool): def _new_conn(self): self.num_connections += 1 return ThredoHTTPConnection(host=self.host, port=self.port) class ThredoHTTPSConnectionPool(HTTPSConnectionPool): def _new_conn(self): self.num_connections += 1 return ThredoHTTPSConnection(host=self.host, port=self.port) class ThredoHTTPConnection(HTTPConnection): def connect(self): """Connect to the host and port specified in __init__.""" # Uses thredo self.sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address) # Important! if self._tunnel_host: self._tunnel() class ThredoHTTPSConnection(HTTPSConnection): def connect(self): """Connect to the host and port specified in __init__.""" # Uses thredo self.sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address) # Important! if self._tunnel_host: server_hostname = self._tunnel_host else: server_hostname = self.host self.sock = self._context.wrap_socket(self.sock, server_hostname=server_hostname) def get_session(): s = requests.Session() s.mount('http://', ThredoAdapter()) s.mount('https://', ThredoAdapter()) return s ================================================ FILE: thredo/signal.py ================================================ # signal.py # # Signal related functionality __all__ = ['SignalEvent'] import curio # -- Thredo from .thr import TAWAIT as AWAIT class SignalEvent: def __init__(self, *args, **kwargs): async def _run(): self._sigevt = curio.SignalEvent(*args, **kwargs) AWAIT(_run) def __del__(self): async def _run(): del self._sigevt AWAIT(_run) def wait(self): AWAIT(self._sigevt.wait) def set(self): AWAIT(self._sigevt.set) ================================================ FILE: thredo/socket.py ================================================ # socket.py # # Standin for the standard socket library. The entire contents of stdlib socket are # made available here. import socket as _socket __all__ = _socket.__all__ from socket import * from socket import _GLOBAL_DEFAULT_TIMEOUT from functools import wraps from . import io import sys @wraps(_socket.socket) def socket(*args, **kwargs): return io.Socket(_socket.socket(*args, **kwargs)) class socket(io.Socket): def __init__(self, *args, **kwargs): super().__init__(_socket.socket(*args, **kwargs)) # @staticmethod # def __new__(cls, *args, **kwargs): # return super().__new__(cls, _socket.socket(*args, **kwargs)) @wraps(_socket.socketpair) def socketpair(*args, **kwargs): s1, s2 = _socket.socketpair(*args, **kwargs) return io.Socket(s1), io.Socket(s2) @wraps(_socket.fromfd) def fromfd(*args, **kwargs): return io.Socket(_socket.fromfd(*args, **kwargs)) # Replacements for blocking functions related to domain names and DNS @wraps(_socket.create_connection) def create_connection(*args, **kwargs): sock = _socket.create_connection(*args, **kwargs) return io.Socket(sock) ================================================ FILE: thredo/sync.py ================================================ # sync.py # # The basic synchronization primitives such as locks, semaphores, and condition variables. __all__ = [ 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition' ] import curio # -- Thredo from .thr import TAWAIT as AWAIT class Event(object): def __init__(self): self._evt = curio.Event() def is_set(self): return self._evt.is_set() def clear(self): self._evt.clear() def wait(self): AWAIT(self._evt.wait) def set(self): AWAIT(self._evt.set) # Base class for all synchronization primitives that operate as context managers. class _LockBase(object): def __enter__(self): self._lock.__enter__() def __exit__(self, *args): self._lock.__exit__(*args) def acquire(self): AWAIT(self._lock.acquire) def release(self): AWAIT(self._lock.release) def locked(self): return self._lock.locked() class Lock(_LockBase): def __init__(self): self._lock = curio.Lock() class RLock(_LockBase): def __init__(self): self._lock = curio.RLock() class Semaphore(_LockBase): def __init__(self): self._lock = curio.Semaphore() @property def value(self): return self._lock.value class BoundedSemaphore(Semaphore): def __init__(self): self._lock = curio.BoundedSemaphore() class Condition(_LockBase): def __init__(self, lock=None): if lock is None: self._lock = curio.Condition(curio.Lock()) else: self._lock = curio.Condition(lock._lock) def locked(self): return self._lock.locked() def wait(self): AWAIT(self._lock.wait) def wait_for(self, predicate): AWAIT(self._lock.wait_for, predicate) def notify(self, n=1): AWAIT(self._lock.notify, n) def notify_all(self): AWAIT(self._lock.notify_all) ================================================ FILE: thredo/thr.py ================================================ # thr.py # # Functions that allow normal threads to promote into Curio async threads. from concurrent.futures import Future from curio.thread import is_async_thread, _locals, AWAIT, AsyncThread from curio import spawn, UniversalQueue _request_queue = None def TAWAIT(coro, *args, **kwargs): ''' Ensure that the caller is an async thread (promoting if necessary), then await for a coroutine ''' if not is_async_thread(): enable_async() return AWAIT(coro, *args, **kwargs) def thread_atexit(callable): _locals.thread_exit.atexit(callable) def enable_async(): ''' Enable asynchronous operations in an existing thread. This only shuts down once the thread exits. ''' if hasattr(_locals, 'thread'): return if _request_queue is None: raise RuntimeError('Curio: enable_threads not used') fut = Future() _request_queue.put(('start', threading.current_thread(), fut)) _locals.thread = fut.result() _locals.thread_exit = ThreadAtExit() def shutdown(thread=_locals.thread, rq=_request_queue): fut = Future() rq.put(('stop', thread, fut)) fut.result() _locals.thread_exit.atexit(shutdown) async def thread_handler(): ''' Special handler function that allows Curio to respond to threads that want to access async functions. This handler must be spawned manually in code that wants to allow normal threads to promote to Curio async threads. ''' global _request_queue assert _request_queue is None, "thread_handler already running" _request_queue = queue.UniversalQueue() try: while True: request, thr, fut = await _request_queue.get() if request == 'start': athread = AsyncThread(None) athread._task = await spawn(athread._coro_runner, daemon=True) athread._thread = thr fut.set_result(athread) elif request == 'stop': thr._request.set_result(None) await thr._task.join() fut.set_result(None) finally: _request_queue = None