From a99e9a91450e5620ca228bf2dee9b23b5abc9ce8 Mon Sep 17 00:00:00 2001 From: monkins1010 <34682781+monkins1010@users.noreply.github.com> Date: Wed, 3 Oct 2018 21:14:56 +0100 Subject: [PATCH] Add files via upload --- thirdparty/README | 30 + thirdparty/asyncio/AUTHORS | 27 + thirdparty/asyncio/COPYING | 201 ++ thirdparty/asyncio/ChangeLog | 338 ++ thirdparty/asyncio/MANIFEST.in | 11 + thirdparty/asyncio/Makefile | 61 + thirdparty/asyncio/README.rst | 101 + thirdparty/asyncio/appveyor.yml | 9 + thirdparty/asyncio/asyncio/__init__.py | 50 + thirdparty/asyncio/asyncio/base_events.py | 1443 +++++++++ thirdparty/asyncio/asyncio/base_subprocess.py | 292 ++ thirdparty/asyncio/asyncio/compat.py | 18 + thirdparty/asyncio/asyncio/constants.py | 7 + thirdparty/asyncio/asyncio/coroutines.py | 328 ++ thirdparty/asyncio/asyncio/events.py | 691 +++++ thirdparty/asyncio/asyncio/futures.py | 478 +++ thirdparty/asyncio/asyncio/locks.py | 478 +++ thirdparty/asyncio/asyncio/log.py | 7 + thirdparty/asyncio/asyncio/proactor_events.py | 549 ++++ thirdparty/asyncio/asyncio/protocols.py | 134 + thirdparty/asyncio/asyncio/queues.py | 253 ++ thirdparty/asyncio/asyncio/selector_events.py | 1141 +++++++ thirdparty/asyncio/asyncio/selectors.py | 611 ++++ thirdparty/asyncio/asyncio/sslproto.py | 690 +++++ thirdparty/asyncio/asyncio/streams.py | 695 +++++ thirdparty/asyncio/asyncio/subprocess.py | 213 ++ thirdparty/asyncio/asyncio/tasks.py | 757 +++++ thirdparty/asyncio/asyncio/test_support.py | 403 +++ thirdparty/asyncio/asyncio/test_utils.py | 503 +++ thirdparty/asyncio/asyncio/transports.py | 306 ++ thirdparty/asyncio/asyncio/unix_events.py | 1064 +++++++ thirdparty/asyncio/asyncio/windows_events.py | 774 +++++ thirdparty/asyncio/asyncio/windows_utils.py | 223 ++ thirdparty/asyncio/check.py | 45 + thirdparty/asyncio/examples/cacheclt.py | 213 ++ thirdparty/asyncio/examples/cachesvr.py | 249 ++ thirdparty/asyncio/examples/child_process.py | 128 + thirdparty/asyncio/examples/crawl.py | 864 ++++++ .../asyncio/examples/echo_client_tulip.py | 20 + .../asyncio/examples/echo_server_tulip.py | 20 + thirdparty/asyncio/examples/fetch0.py | 35 + thirdparty/asyncio/examples/fetch1.py | 78 + thirdparty/asyncio/examples/fetch2.py | 141 + thirdparty/asyncio/examples/fetch3.py | 230 ++ .../asyncio/examples/fuzz_as_completed.py | 69 + thirdparty/asyncio/examples/hello_callback.py | 17 + .../asyncio/examples/hello_coroutine.py | 18 + thirdparty/asyncio/examples/qspeed.py | 43 + thirdparty/asyncio/examples/shell.py | 52 + .../asyncio/examples/simple_tcp_server.py | 154 + thirdparty/asyncio/examples/sink.py | 94 + thirdparty/asyncio/examples/source.py | 100 + thirdparty/asyncio/examples/source1.py | 98 + thirdparty/asyncio/examples/stacks.py | 44 + .../examples/subprocess_attach_read_pipe.py | 33 + .../examples/subprocess_attach_write_pipe.py | 35 + .../asyncio/examples/subprocess_shell.py | 87 + thirdparty/asyncio/examples/tcp_echo.py | 128 + .../asyncio/examples/timing_tcp_server.py | 168 + thirdparty/asyncio/examples/udp_echo.py | 104 + thirdparty/asyncio/overlapped.c | 1346 ++++++++ thirdparty/asyncio/pypi.bat | 1 + thirdparty/asyncio/release.py | 516 ++++ thirdparty/asyncio/run_aiotest.py | 14 + thirdparty/asyncio/runtests.py | 312 ++ thirdparty/asyncio/setup.py | 53 + thirdparty/asyncio/tests/echo.py | 8 + thirdparty/asyncio/tests/echo2.py | 6 + thirdparty/asyncio/tests/echo3.py | 11 + thirdparty/asyncio/tests/keycert3.pem | 73 + thirdparty/asyncio/tests/pycacert.pem | 78 + thirdparty/asyncio/tests/sample.crt | 14 + thirdparty/asyncio/tests/sample.key | 15 + thirdparty/asyncio/tests/ssl_cert.pem | 15 + thirdparty/asyncio/tests/ssl_key.pem | 16 + thirdparty/asyncio/tests/test_base_events.py | 1716 ++++++++++ thirdparty/asyncio/tests/test_events.py | 2747 +++++++++++++++++ thirdparty/asyncio/tests/test_futures.py | 551 ++++ thirdparty/asyncio/tests/test_locks.py | 921 ++++++ thirdparty/asyncio/tests/test_pep492.py | 232 ++ .../asyncio/tests/test_proactor_events.py | 594 ++++ thirdparty/asyncio/tests/test_queues.py | 626 ++++ .../asyncio/tests/test_selector_events.py | 1793 +++++++++++ thirdparty/asyncio/tests/test_selectors.py | 454 +++ thirdparty/asyncio/tests/test_sslproto.py | 89 + thirdparty/asyncio/tests/test_streams.py | 850 +++++ thirdparty/asyncio/tests/test_subprocess.py | 501 +++ thirdparty/asyncio/tests/test_tasks.py | 2341 ++++++++++++++ thirdparty/asyncio/tests/test_transports.py | 91 + thirdparty/asyncio/tests/test_unix_events.py | 1584 ++++++++++ .../asyncio/tests/test_windows_events.py | 162 + .../asyncio/tests/test_windows_utils.py | 182 ++ thirdparty/asyncio/tox.ini | 21 + thirdparty/asyncio/update_asyncio.sh | 25 + thirdparty/asyncio/update_stdlib.sh | 78 + 95 files changed, 34189 insertions(+) create mode 100644 thirdparty/README create mode 100644 thirdparty/asyncio/AUTHORS create mode 100644 thirdparty/asyncio/COPYING create mode 100644 thirdparty/asyncio/ChangeLog create mode 100644 thirdparty/asyncio/MANIFEST.in create mode 100644 thirdparty/asyncio/Makefile create mode 100644 thirdparty/asyncio/README.rst create mode 100644 thirdparty/asyncio/appveyor.yml create mode 100644 thirdparty/asyncio/asyncio/__init__.py create mode 100644 thirdparty/asyncio/asyncio/base_events.py create mode 100644 thirdparty/asyncio/asyncio/base_subprocess.py create mode 100644 thirdparty/asyncio/asyncio/compat.py create mode 100644 thirdparty/asyncio/asyncio/constants.py create mode 100644 thirdparty/asyncio/asyncio/coroutines.py create mode 100644 thirdparty/asyncio/asyncio/events.py create mode 100644 thirdparty/asyncio/asyncio/futures.py create mode 100644 thirdparty/asyncio/asyncio/locks.py create mode 100644 thirdparty/asyncio/asyncio/log.py create mode 100644 thirdparty/asyncio/asyncio/proactor_events.py create mode 100644 thirdparty/asyncio/asyncio/protocols.py create mode 100644 thirdparty/asyncio/asyncio/queues.py create mode 100644 thirdparty/asyncio/asyncio/selector_events.py create mode 100644 thirdparty/asyncio/asyncio/selectors.py create mode 100644 thirdparty/asyncio/asyncio/sslproto.py create mode 100644 thirdparty/asyncio/asyncio/streams.py create mode 100644 thirdparty/asyncio/asyncio/subprocess.py create mode 100644 thirdparty/asyncio/asyncio/tasks.py create mode 100644 thirdparty/asyncio/asyncio/test_support.py create mode 100644 thirdparty/asyncio/asyncio/test_utils.py create mode 100644 thirdparty/asyncio/asyncio/transports.py create mode 100644 thirdparty/asyncio/asyncio/unix_events.py create mode 100644 thirdparty/asyncio/asyncio/windows_events.py create mode 100644 thirdparty/asyncio/asyncio/windows_utils.py create mode 100644 thirdparty/asyncio/check.py create mode 100644 thirdparty/asyncio/examples/cacheclt.py create mode 100644 thirdparty/asyncio/examples/cachesvr.py create mode 100644 thirdparty/asyncio/examples/child_process.py create mode 100644 thirdparty/asyncio/examples/crawl.py create mode 100644 thirdparty/asyncio/examples/echo_client_tulip.py create mode 100644 thirdparty/asyncio/examples/echo_server_tulip.py create mode 100644 thirdparty/asyncio/examples/fetch0.py create mode 100644 thirdparty/asyncio/examples/fetch1.py create mode 100644 thirdparty/asyncio/examples/fetch2.py create mode 100644 thirdparty/asyncio/examples/fetch3.py create mode 100644 thirdparty/asyncio/examples/fuzz_as_completed.py create mode 100644 thirdparty/asyncio/examples/hello_callback.py create mode 100644 thirdparty/asyncio/examples/hello_coroutine.py create mode 100644 thirdparty/asyncio/examples/qspeed.py create mode 100644 thirdparty/asyncio/examples/shell.py create mode 100644 thirdparty/asyncio/examples/simple_tcp_server.py create mode 100644 thirdparty/asyncio/examples/sink.py create mode 100644 thirdparty/asyncio/examples/source.py create mode 100644 thirdparty/asyncio/examples/source1.py create mode 100644 thirdparty/asyncio/examples/stacks.py create mode 100644 thirdparty/asyncio/examples/subprocess_attach_read_pipe.py create mode 100644 thirdparty/asyncio/examples/subprocess_attach_write_pipe.py create mode 100644 thirdparty/asyncio/examples/subprocess_shell.py create mode 100644 thirdparty/asyncio/examples/tcp_echo.py create mode 100644 thirdparty/asyncio/examples/timing_tcp_server.py create mode 100644 thirdparty/asyncio/examples/udp_echo.py create mode 100644 thirdparty/asyncio/overlapped.c create mode 100644 thirdparty/asyncio/pypi.bat create mode 100644 thirdparty/asyncio/release.py create mode 100644 thirdparty/asyncio/run_aiotest.py create mode 100644 thirdparty/asyncio/runtests.py create mode 100644 thirdparty/asyncio/setup.py create mode 100644 thirdparty/asyncio/tests/echo.py create mode 100644 thirdparty/asyncio/tests/echo2.py create mode 100644 thirdparty/asyncio/tests/echo3.py create mode 100644 thirdparty/asyncio/tests/keycert3.pem create mode 100644 thirdparty/asyncio/tests/pycacert.pem create mode 100644 thirdparty/asyncio/tests/sample.crt create mode 100644 thirdparty/asyncio/tests/sample.key create mode 100644 thirdparty/asyncio/tests/ssl_cert.pem create mode 100644 thirdparty/asyncio/tests/ssl_key.pem create mode 100644 thirdparty/asyncio/tests/test_base_events.py create mode 100644 thirdparty/asyncio/tests/test_events.py create mode 100644 thirdparty/asyncio/tests/test_futures.py create mode 100644 thirdparty/asyncio/tests/test_locks.py create mode 100644 thirdparty/asyncio/tests/test_pep492.py create mode 100644 thirdparty/asyncio/tests/test_proactor_events.py create mode 100644 thirdparty/asyncio/tests/test_queues.py create mode 100644 thirdparty/asyncio/tests/test_selector_events.py create mode 100644 thirdparty/asyncio/tests/test_selectors.py create mode 100644 thirdparty/asyncio/tests/test_sslproto.py create mode 100644 thirdparty/asyncio/tests/test_streams.py create mode 100644 thirdparty/asyncio/tests/test_subprocess.py create mode 100644 thirdparty/asyncio/tests/test_tasks.py create mode 100644 thirdparty/asyncio/tests/test_transports.py create mode 100644 thirdparty/asyncio/tests/test_unix_events.py create mode 100644 thirdparty/asyncio/tests/test_windows_events.py create mode 100644 thirdparty/asyncio/tests/test_windows_utils.py create mode 100644 thirdparty/asyncio/tox.ini create mode 100644 thirdparty/asyncio/update_asyncio.sh create mode 100644 thirdparty/asyncio/update_stdlib.sh diff --git a/thirdparty/README b/thirdparty/README new file mode 100644 index 0000000..5fd5911 --- /dev/null +++ b/thirdparty/README @@ -0,0 +1,30 @@ +"asyncio" is a clone of https://github.com/python/asyncio as of revision +07ac834068037d8206d6c941e029474bda8e08f2 (03 Nov 2016) with the following +patch: + +--- asyncio/base_events.py 2016-11-07 16:45:34.587238061 -0600 ++++ asyncio/base_events.py 2016-11-07 14:50:34.169035083 -0600 +@@ -40,6 +40,11 @@ + + __all__ = ['BaseEventLoop'] + ++# lhl: We need to add this back in because 3.3 doesn't have a default max_workers in ThreadPoolExecuter ++# https://bugs.python.org/issue26796 ++# https://docs.python.org/3/library/asyncio-eventloop.html#executor ++# https://docs.python.org/3/library/concurrent.futures.html ++_MAX_WORKERS=8 + + # Minimum number of _scheduled timer handles before cleanup of + # cancelled handles is performed. +@@ -619,7 +624,10 @@ + if executor is None: + executor = self._default_executor + if executor is None: +- executor = concurrent.futures.ThreadPoolExecutor() ++ try: ++ executor = concurrent.futures.ThreadPoolExecutor() ++ except: ++ executor = concurrent.futures.ThreadPoolExecutor(_MAX_WORKERS) + self._default_executor = executor + return futures.wrap_future(executor.submit(func, *args), loop=self) + diff --git a/thirdparty/asyncio/AUTHORS b/thirdparty/asyncio/AUTHORS new file mode 100644 index 0000000..c16bdd4 --- /dev/null +++ b/thirdparty/asyncio/AUTHORS @@ -0,0 +1,27 @@ +A. Jesse Jiryu Davis +Aaron Griffith +Andrew Svetlov +Anthony Baire +Antoine Pitrou +Arnaud Faure +Aymeric Augustin +Brett Cannon +Charles-François Natali +Christian Heimes +Donald Stufft +Eli Bendersky +Geert Jansen +Giampaolo Rodola' +Guido van Rossum : creator of the asyncio project and author of the PEP 3156 +Gustavo Carneiro +Jeff Quast +Jonathan Slenders +Nikolay Kim +Richard Oudkerk +Saúl Ibarra Corretgé +Serhiy Storchaka +Vajrasky Kok +Victor Stinner +Vladimir Kryachko +Yann Sionneau +Yury Selivanov diff --git a/thirdparty/asyncio/COPYING b/thirdparty/asyncio/COPYING new file mode 100644 index 0000000..11069ed --- /dev/null +++ b/thirdparty/asyncio/COPYING @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/thirdparty/asyncio/ChangeLog b/thirdparty/asyncio/ChangeLog new file mode 100644 index 0000000..421704c --- /dev/null +++ b/thirdparty/asyncio/ChangeLog @@ -0,0 +1,338 @@ +Tulip 3.4.4 +=========== + +* Issue #234: Drop asyncio.JoinableQueue on Python 3.5 and newer + + +2015-02-04: Tulip 3.4.3 +======================= + +Major changes +------------- + +* New SSL implementation using ssl.MemoryBIO. The new implementation requires + Python 3.5 and newer, otherwise the legacy implementation is used. +* On Python 3.5 and newer usable, the ProactorEventLoop now supports SSL + thanks to the new SSL implementation. +* Fix multiple resource leaks: close sockets on error, explicitly clear + references, emit ResourceWarning when event loops and transports are not + closed explicitly, etc. +* The proactor event loop is now much more reliable (no more known race + condition). +* Enhance handling of task cancellation. + +Changes of the asyncio API +-------------------------- + +* Export BaseEventLoop symbol in the asyncio namespace +* create_task(), call_soon(), call_soon_threadsafe(), call_later(), + call_at() and run_in_executor() methods of BaseEventLoop now raise an + exception if the event loop is closed. +* call_soon(), call_soon_threadsafe(), call_later(), call_at() and + run_in_executor() methods of BaseEventLoop now raise an exception if the + callback is a coroutine object. +* BaseEventLoopPolicy.get_event_loop() now always raises a RuntimeError + if there is no event loop in the curren thread, instead of using an + assertion (which can be disabld at runtime) and so raises an AssertionError. +* selectors: Selector.get_key() now raises an exception if the selector is + closed. +* If wait_for() is cancelled, the waited task is also cancelled. +* _UnixSelectorEventLoop.add_signal_handler() now raises an exception if + the callback is a coroutine object or a coroutine function. It also raises + an exception if the event loop is closed. + +Performances +------------ + +* sock_connect() doesn't check if the address is already resolved anymore. + The check is only done in debug mode. Moreover, the check uses inet_pton() + instead of getaddrinfo(), if inet_pton() is available, because getaddrinfo() + is slow (around 10 us per call). + +Debug +----- + +* Better repr() of _ProactorBasePipeTransport, _SelectorTransport, + _UnixReadPipeTransport and _UnixWritePipeTransport: add closed/closing + status and the file descriptor +* Add repr(PipeHandle) +* PipeHandle destructor now emits a ResourceWarning is the pipe is not closed + explicitly. +* In debug mode, call_at() method of BaseEventLoop now raises an exception + if called from the wrong thread (not from the thread running the event + loop). Before, it only raised an exception if current thread had an event + loop. +* A ResourceWarning is now emitted when event loops and transports are + destroyed before being closed. +* BaseEventLoop.call_exception_handler() now logs the traceback where + the current handle was created (if no source_traceback was specified). +* BaseSubprocessTransport.close() now logs a warning when the child process is + still running and the method kills it. + +Bug fixes +--------- + +* windows_utils.socketpair() now reuses socket.socketpair() if available + (Python 3.5 or newer). +* Fix IocpProactor.accept_pipe(): handle ERROR_PIPE_CONNECTED, it means + that the pipe is connected. _overlapped.Overlapped.ConnectNamedPipe() now + returns True on ERROR_PIPE_CONNECTED. +* Rewrite IocpProactor.connect_pipe() using polling to avoid tricky bugs + if the connection is cancelled, instead of using QueueUserWorkItem() to run + blocking code. +* Fix IocpProactor.recv(): handle BrokenPipeError, set the result to an empty + string. +* Fix ProactorEventLoop.start_serving_pipe(): if a client connected while the + server is closing, drop the client connection. +* Fix a tricky race condition when IocpProactor.wait_for_handle() is + cancelled: wait until the wait is really cancelled before destroying the + overlapped object. Unregister also the overlapped operation to not block + in IocpProactor.close() before the wait will never complete. +* Fix _UnixSubprocessTransport._start(): make the write end of the stdin pipe + non-inheritable. +* Set more attributes in the body of classes to avoid attribute errors in + destructors if an error occurred in the constructor. +* Fix SubprocessStreamProtocol.process_exited(): close the transport + and clear its reference to the transport. +* Fix SubprocessStreamProtocol.connection_made(): set the transport of + stdout and stderr streams to respect reader buffer limits (stop reading when + the buffer is full). +* Fix FlowControlMixin constructor: if the loop parameter is None, get the + current event loop. +* Fix selectors.EpollSelector.select(): don't fail anymore if no file + descriptor is registered. +* Fix _SelectorTransport: don't wakeup the waiter if it was cancelled +* Fix _SelectorTransport._call_connection_lost(): only call connection_lost() + if connection_made() was already called. +* Fix BaseSelectorEventLoop._accept_connection(): close the transport on + error. In debug mode, log errors (ex: SSL handshake failure) on the creation + of the transport for incoming connection. +* Fix BaseProactorEventLoop.close(): stop the proactor before closing the + event loop because stopping the proactor may schedule new callbacks, which + is now forbidden when the event loop is closed. +* Fix wrap_future() to not use a free variable and so not keep a frame alive + too long. +* Fix formatting of the "Future/Task exception was never retrieved" log: add + a newline before the traceback. +* WriteSubprocessPipeProto.connection_lost() now clears its reference to the + subprocess.Popen object. +* If the creation of a subprocess transport fails, the child process is killed + and the event loop waits asynchronously for its completion. +* BaseEventLoop.run_until_complete() now consumes the exception to not log a + warning when a BaseException like KeyboardInterrupt is raised and + run_until_complete() is not a future (but a coroutine object). +* create_connection(), create_datagram_endpoint(), connect_read_pipe() and + connect_write_pipe() methods of BaseEventLoop now close the transport on + error. + +Other changes +------------- + +* Add tox.ini to run tests using tox. +* _FlowControlMixin constructor now requires an event loop. +* Embed asyncio/test_support.py to not depend on test.support of the system + Python. For example, test.support is not installed by default on Windows. +* selectors.Selector.close() now clears its reference to the mapping object. +* _SelectorTransport and _UnixWritePipeTransport now only starts listening for + read events after protocol.connection_made() has been called +* _SelectorTransport._fatal_error() now only logs ConnectionAbortedError + in debug mode. +* BaseProactorEventLoop._loop_self_reading() now handles correctly + CancelledError (just exit) and logs an error for other exceptions. +* _ProactorBasePipeTransport now clears explicitly references to read and + write future and to the socket +* BaseSubprocessTransport constructor now calls the internal _connect_pipes() + method (previously called _post_init()). The constructor now accepts an + optional waiter parameter to notify when the transport is ready. +* send_signal(), terminate() and kill() methods of BaseSubprocessTransport now + raise a ProcessLookupError if the process already exited. +* Add run_aiotest.py to run the aiotest test suite +* Add release.py script to build wheel packages on Windows and run unit tests + + +2014-09-30: Tulip 3.4.2 +======================= + +New shiny methods like create_task(), better documentation, much better debug +mode, better tests. + +asyncio API +----------- + +* Add BaseEventLoop.create_task() method: schedule a coroutine object. + It allows other asyncio implementations to use their own Task class to + change its behaviour. + +* New BaseEventLoop methods: + + - create_task(): schedule a coroutine + - get_debug() + - is_closed() + - set_debug() + +* Add _FlowControlMixin.get_write_buffer_limits() method + +* sock_recv(), sock_sendall(), sock_connect(), sock_accept() methods of + SelectorEventLoop now raise an exception if the socket is blocking mode + +* Include unix_events/windows_events symbols in asyncio.__all__. + Examples: SelectorEventLoop, ProactorEventLoop, DefaultEventLoopPolicy. + +* attach(), detach(), loop, active_count and waiters attributes of the Server + class are now private + +* BaseEventLoop: run_forever(), run_until_complete() now raises an exception if + the event loop was closed + +* close() now raises an exception if the event loop is running, because pending + callbacks would be lost + +* Queue now accepts a float for the maximum size. + +* Process.communicate() now ignores BrokenPipeError and ConnectionResetError + exceptions, as Popen.communicate() of the subprocess module + + +Performances +------------ + +* Optimize handling of cancelled timers + + +Debug +----- + +* Future (and Task), CoroWrapper and Handle now remembers where they were + created (new _source_traceback object), traceback displayed when errors are + logged. + +* On Python 3.4 and newer, Task destrutor now logs a warning if the task was + destroyed while it was still pending. It occurs if the last reference + to the task was removed, while the coroutine didn't finish yet. + +* Much more useful events are logged: + + - Event loop closed + - Network connection + - Creation of a subprocess + - Pipe lost + - Log many errors previously silently ignored + - SSL handshake failure + - etc. + +* BaseEventLoop._debug is now True if the envrionement variable + PYTHONASYNCIODEBUG is set + +* Log the duration of DNS resolution and SSL handshake + +* Log a warning if a callback blocks the event loop longer than 100 ms + (configurable duration) + +* repr(CoroWrapper) and repr(Task) now contains the current status of the + coroutine (running, done), current filename and line number, and filename and + line number where the object was created + +* Enhance representation (repr) of transports: add the file descriptor, status + (idle, polling, writing, etc.), size of the write buffer, ... + +* Add repr(BaseEventLoop) + +* run_until_complete() doesn't log a warning anymore when called with a + coroutine object which raises an exception. + + +Bugfixes +-------- + +* windows_utils.socketpair() now ensures that sockets are closed in case + of error. + +* Rewrite bricks of the IocpProactor() to make it more reliable + +* IocpProactor destructor now closes it. + +* _OverlappedFuture.set_exception() now cancels the overlapped operation. + +* Rewrite _WaitHandleFuture: + + - cancel() is now able to signal the cancellation to the overlapped object + - _unregister_wait() now catchs and logs exceptions + +* PipeServer.close() (class used on Windows) now cancels the accept pipe + future. + +* Rewrite signal handling in the UNIX implementation of SelectorEventLoop: + use the self-pipe to store pending signals instead of registering a + signal handler calling directly _handle_signal(). The change fixes a + race condition. + +* create_unix_server(): close the socket on error. + +* Fix wait_for() + +* Rewrite gather() + +* drain() is now a classic coroutine, no more special return value (empty + tuple) + +* Rewrite SelectorEventLoop.sock_connect() to handle correctly timeout + +* Process data of the self-pipe faster to accept more pending events, + especially signals written by signal handlers: the callback reads all pending + data, not only a single byte + +* Don't try to set the result of a Future anymore if it was cancelled + (explicitly or by a timeout) + +* CoroWrapper now works around CPython issue #21209: yield from & custom + generator classes don't work together, issue with the send() method. It only + affected asyncio in debug mode on Python older than 3.4.2 + + +Misc changes +------------ + +* windows_utils.socketpair() now supports IPv6. + +* Better documentation (online & docstrings): fill remaining XXX, more examples + +* new asyncio.coroutines submodule, to ease maintenance with the trollius + project: @coroutine, _DEBUG, iscoroutine() and iscoroutinefunction() have + been moved from asyncio.tasks to asyncio.coroutines + +* Cleanup code, ex: remove unused attribute (ex: _rawsock) + +* Reuse os.set_blocking() of Python 3.5. + +* Close explicitly the event loop in Tulip examples. + +* runtests.py now mention if tests are running in release or debug mode. + + +2014-05-19: Tulip 3.4.1 +======================= + +2014-02-24: Tulip 0.4.1 +======================= + +2014-02-10: Tulip 0.3.1 +======================= + +* Add asyncio.subprocess submodule and the Process class. + +2013-11-25: Tulip 0.2.1 +======================= + +* Add support of subprocesses using transports and protocols. + +2013-10-22: Tulip 0.1.1 +======================= + +* First release. + +Creation of the project +======================= + +* 2013-10-14: The tulip package was renamed to asyncio. +* 2012-10-16: Creation of the Tulip project, started as mail threads on the + python-ideas mailing list. diff --git a/thirdparty/asyncio/MANIFEST.in b/thirdparty/asyncio/MANIFEST.in new file mode 100644 index 0000000..d0dbde1 --- /dev/null +++ b/thirdparty/asyncio/MANIFEST.in @@ -0,0 +1,11 @@ +include AUTHORS COPYING +include Makefile +include overlapped.c pypi.bat +include check.py runtests.py run_aiotest.py release.py +include update_stdlib.sh + +recursive-include examples *.py +recursive-include tests *.crt +recursive-include tests *.key +recursive-include tests *.pem +recursive-include tests *.py diff --git a/thirdparty/asyncio/Makefile b/thirdparty/asyncio/Makefile new file mode 100644 index 0000000..3ae000e --- /dev/null +++ b/thirdparty/asyncio/Makefile @@ -0,0 +1,61 @@ +# Some simple testing tasks (sorry, UNIX only). + +PYTHON=python3 +VERBOSE=$(V) +V= 0 +FLAGS= + +test: + $(PYTHON) runtests.py -v $(VERBOSE) $(FLAGS) + PYTHONASYNCIODEBUG=1 $(PYTHON) runtests.py -v $(VERBOSE) $(FLAGS) + +vtest: + $(PYTHON) runtests.py -v 1 $(FLAGS) + +testloop: + while sleep 1; do $(PYTHON) runtests.py -v $(VERBOSE) $(FLAGS); done + +# See runtests.py for coverage installation instructions. +cov coverage: + $(PYTHON) runtests.py --coverage -v $(VERBOSE) $(FLAGS) + +check: + $(PYTHON) check.py + +# Requires "pip install pep8". +pep8: check + pep8 --ignore E125,E127,E226 tests asyncio + +clean: + rm -rf `find . -name __pycache__` + rm -f `find . -type f -name '*.py[co]' ` + rm -f `find . -type f -name '*~' ` + rm -f `find . -type f -name '.*~' ` + rm -f `find . -type f -name '@*' ` + rm -f `find . -type f -name '#*#' ` + rm -f `find . -type f -name '*.orig' ` + rm -f `find . -type f -name '*.rej' ` + rm -rf dist + rm -f .coverage + rm -rf htmlcov + rm -rf build + rm -rf asyncio.egg-info + rm -f MANIFEST + + +# For distribution builders only! +# Push a source distribution for Python 3.3 to PyPI. +# You must update the version in setup.py first. +# A PyPI user configuration in ~/.pypirc is required; +# you can create a suitable confifuration using +# python setup.py register +pypi: clean + python3.3 setup.py sdist upload + +# The corresponding action on Windows is pypi.bat. For that to work, +# you need to install wheel and setuptools. The easiest way is to get +# pip using the get-pip.py script found here: +# https://pip.pypa.io/en/latest/installing.html#install-pip +# That will install setuptools and pip; then you can just do +# \Python33\python.exe -m pip install wheel +# after which the pypi.bat script should work. diff --git a/thirdparty/asyncio/README.rst b/thirdparty/asyncio/README.rst new file mode 100644 index 0000000..9e95903 --- /dev/null +++ b/thirdparty/asyncio/README.rst @@ -0,0 +1,101 @@ +.. image:: https://travis-ci.org/python/asyncio.svg?branch=master + :target: https://travis-ci.org/python/asyncio + +.. image:: https://ci.appveyor.com/api/projects/status/u72781t69ljdpm2y?svg=true + :target: https://ci.appveyor.com/project/1st1/asyncio + + +The asyncio module provides infrastructure for writing single-threaded +concurrent code using coroutines, multiplexing I/O access over sockets and +other resources, running network clients and servers, and other related +primitives. Here is a more detailed list of the package contents: + +* a pluggable event loop with various system-specific implementations; + +* transport and protocol abstractions (similar to those in Twisted); + +* concrete support for TCP, UDP, SSL, subprocess pipes, delayed calls, and + others (some may be system-dependent); + +* a Future class that mimics the one in the concurrent.futures module, but + adapted for use with the event loop; + +* coroutines and tasks based on ``yield from`` (PEP 380), to help write + concurrent code in a sequential fashion; + +* cancellation support for Futures and coroutines; + +* synchronization primitives for use between coroutines in a single thread, + mimicking those in the threading module; + +* an interface for passing work off to a threadpool, for times when you + absolutely, positively have to use a library that makes blocking I/O calls. + +Note: The implementation of asyncio was previously called "Tulip". + + +Installation +============ + +To install asyncio, type:: + + pip install asyncio + +asyncio requires Python 3.3 or later! The asyncio module is part of the Python +standard library since Python 3.4. + +asyncio is a free software distributed under the Apache license version 2.0. + + +Websites +======== + +* `asyncio project at GitHub `_: source + code, bug tracker +* `asyncio documentation `_ +* Mailing list: `python-tulip Google Group + `_ +* IRC: join the ``#asyncio`` channel on the Freenode network + + +Development +=========== + +The actual code lives in the 'asyncio' subdirectory. Tests are in the 'tests' +subdirectory. + +To run tests, run:: + + tox + +Or use the Makefile:: + + make test + +To run coverage (coverage package is required):: + + make coverage + +On Windows, things are a little more complicated. Assume ``P`` is your +Python binary (for example ``C:\Python33\python.exe``). + +You must first build the _overlapped.pyd extension and have it placed +in the asyncio directory, as follows:: + + C:\> P setup.py build_ext --inplace + +If this complains about vcvars.bat, you probably don't have the +required version of Visual Studio installed. Compiling extensions for +Python 3.3 requires Microsoft Visual C++ 2010 (MSVC 10.0) of any +edition; you can download Visual Studio Express 2010 for free from +http://www.visualstudio.com/downloads (scroll down to Visual C++ 2010 +Express). + +Once you have built the _overlapped.pyd extension successfully you can +run the tests as follows:: + + C:\> P runtests.py + +And coverage as follows:: + + C:\> P runtests.py --coverage diff --git a/thirdparty/asyncio/appveyor.yml b/thirdparty/asyncio/appveyor.yml new file mode 100644 index 0000000..607c723 --- /dev/null +++ b/thirdparty/asyncio/appveyor.yml @@ -0,0 +1,9 @@ +environment: + matrix: + - PYTHON: "C:\\Python35" + - PYTHON: "C:\\Python35-x64" + +build: false + +test_script: + - "%PYTHON%\\python.exe runtests.py" diff --git a/thirdparty/asyncio/asyncio/__init__.py b/thirdparty/asyncio/asyncio/__init__.py new file mode 100644 index 0000000..011466b --- /dev/null +++ b/thirdparty/asyncio/asyncio/__init__.py @@ -0,0 +1,50 @@ +"""The asyncio package, tracking PEP 3156.""" + +import sys + +# The selectors module is in the stdlib in Python 3.4 but not in 3.3. +# Do this first, so the other submodules can use "from . import selectors". +# Prefer asyncio/selectors.py over the stdlib one, as ours may be newer. +try: + from . import selectors +except ImportError: + import selectors # Will also be exported. + +if sys.platform == 'win32': + # Similar thing for _overlapped. + try: + from . import _overlapped + except ImportError: + import _overlapped # Will also be exported. + +# This relies on each of the submodules having an __all__ variable. +from .base_events import * +from .coroutines import * +from .events import * +from .futures import * +from .locks import * +from .protocols import * +from .queues import * +from .streams import * +from .subprocess import * +from .tasks import * +from .transports import * + +__all__ = (base_events.__all__ + + coroutines.__all__ + + events.__all__ + + futures.__all__ + + locks.__all__ + + protocols.__all__ + + queues.__all__ + + streams.__all__ + + subprocess.__all__ + + tasks.__all__ + + transports.__all__) + +if sys.platform == 'win32': # pragma: no cover + from .windows_events import * + __all__ += windows_events.__all__ +else: + from .unix_events import * # pragma: no cover + __all__ += unix_events.__all__ diff --git a/thirdparty/asyncio/asyncio/base_events.py b/thirdparty/asyncio/asyncio/base_events.py new file mode 100644 index 0000000..19d49b1 --- /dev/null +++ b/thirdparty/asyncio/asyncio/base_events.py @@ -0,0 +1,1443 @@ +"""Base implementation of event loop. + +The event loop can be broken up into a multiplexer (the part +responsible for notifying us of I/O events) and the event loop proper, +which wraps a multiplexer with functionality for scheduling callbacks, +immediately or at a given time in the future. + +Whenever a public API takes a callback, subsequent positional +arguments will be passed to the callback if/when it is called. This +avoids the proliferation of trivial lambdas implementing closures. +Keyword arguments for the callback are not supported; this is a +conscious design decision, leaving the door open for keyword arguments +to modify the meaning of the API call itself. +""" + +import collections +import concurrent.futures +import heapq +import inspect +import itertools +import logging +import os +import socket +import subprocess +import threading +import time +import traceback +import sys +import warnings +import weakref + +from . import compat +from . import coroutines +from . import events +from . import futures +from . import tasks +from .coroutines import coroutine +from .log import logger + + +__all__ = ['BaseEventLoop'] + +# lhl: We need to add this back in because 3.3 doesn't have a default max_workers in ThreadPoolExecuter +# https://bugs.python.org/issue26796 +# https://docs.python.org/3/library/asyncio-eventloop.html#executor +# https://docs.python.org/3/library/concurrent.futures.html +_MAX_WORKERS=8 + +# Minimum number of _scheduled timer handles before cleanup of +# cancelled handles is performed. +_MIN_SCHEDULED_TIMER_HANDLES = 100 + +# Minimum fraction of _scheduled timer handles that are cancelled +# before cleanup of cancelled handles is performed. +_MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5 + +# Exceptions which must not call the exception handler in fatal error +# methods (_fatal_error()) +_FATAL_ERROR_IGNORE = (BrokenPipeError, + ConnectionResetError, ConnectionAbortedError) + + +def _format_handle(handle): + cb = handle._callback + if inspect.ismethod(cb) and isinstance(cb.__self__, tasks.Task): + # format the task + return repr(cb.__self__) + else: + return str(handle) + + +def _format_pipe(fd): + if fd == subprocess.PIPE: + return '' + elif fd == subprocess.STDOUT: + return '' + else: + return repr(fd) + + +def _set_reuseport(sock): + if not hasattr(socket, 'SO_REUSEPORT'): + raise ValueError('reuse_port not supported by socket module') + else: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + except OSError: + raise ValueError('reuse_port not supported by socket module, ' + 'SO_REUSEPORT defined but not implemented.') + + +# Linux's sock.type is a bitmask that can include extra info about socket. +_SOCKET_TYPE_MASK = 0 +if hasattr(socket, 'SOCK_NONBLOCK'): + _SOCKET_TYPE_MASK |= socket.SOCK_NONBLOCK +if hasattr(socket, 'SOCK_CLOEXEC'): + _SOCKET_TYPE_MASK |= socket.SOCK_CLOEXEC + + +def _ipaddr_info(host, port, family, type, proto): + # Try to skip getaddrinfo if "host" is already an IP. Users might have + # handled name resolution in their own code and pass in resolved IPs. + if not hasattr(socket, 'inet_pton'): + return + + if proto not in {0, socket.IPPROTO_TCP, socket.IPPROTO_UDP} or \ + host is None: + return None + + type &= ~_SOCKET_TYPE_MASK + if type == socket.SOCK_STREAM: + proto = socket.IPPROTO_TCP + elif type == socket.SOCK_DGRAM: + proto = socket.IPPROTO_UDP + else: + return None + + if port is None: + port = 0 + elif isinstance(port, bytes) and port == b'': + port = 0 + elif isinstance(port, str) and port == '': + port = 0 + else: + # If port's a service name like "http", don't skip getaddrinfo. + try: + port = int(port) + except (TypeError, ValueError): + return None + + if family == socket.AF_UNSPEC: + afs = [socket.AF_INET, socket.AF_INET6] + else: + afs = [family] + + if isinstance(host, bytes): + host = host.decode('idna') + if '%' in host: + # Linux's inet_pton doesn't accept an IPv6 zone index after host, + # like '::1%lo0'. + return None + + for af in afs: + try: + socket.inet_pton(af, host) + # The host has already been resolved. + return af, type, proto, '', (host, port) + except OSError: + pass + + # "host" is not an IP address. + return None + + +def _ensure_resolved(address, *, family=0, type=socket.SOCK_STREAM, proto=0, + flags=0, loop): + host, port = address[:2] + info = _ipaddr_info(host, port, family, type, proto) + if info is not None: + # "host" is already a resolved IP. + fut = loop.create_future() + fut.set_result([info]) + return fut + else: + return loop.getaddrinfo(host, port, family=family, type=type, + proto=proto, flags=flags) + + +def _run_until_complete_cb(fut): + exc = fut._exception + if (isinstance(exc, BaseException) + and not isinstance(exc, Exception)): + # Issue #22429: run_forever() already finished, no need to + # stop it. + return + fut._loop.stop() + + +class Server(events.AbstractServer): + + def __init__(self, loop, sockets): + self._loop = loop + self.sockets = sockets + self._active_count = 0 + self._waiters = [] + + def __repr__(self): + return '<%s sockets=%r>' % (self.__class__.__name__, self.sockets) + + def _attach(self): + assert self.sockets is not None + self._active_count += 1 + + def _detach(self): + assert self._active_count > 0 + self._active_count -= 1 + if self._active_count == 0 and self.sockets is None: + self._wakeup() + + def close(self): + sockets = self.sockets + if sockets is None: + return + self.sockets = None + for sock in sockets: + self._loop._stop_serving(sock) + if self._active_count == 0: + self._wakeup() + + def _wakeup(self): + waiters = self._waiters + self._waiters = None + for waiter in waiters: + if not waiter.done(): + waiter.set_result(waiter) + + @coroutine + def wait_closed(self): + if self.sockets is None or self._waiters is None: + return + waiter = self._loop.create_future() + self._waiters.append(waiter) + yield from waiter + + +class BaseEventLoop(events.AbstractEventLoop): + + def __init__(self): + self._timer_cancelled_count = 0 + self._closed = False + self._stopping = False + self._ready = collections.deque() + self._scheduled = [] + self._default_executor = None + self._internal_fds = 0 + # Identifier of the thread running the event loop, or None if the + # event loop is not running + self._thread_id = None + self._clock_resolution = time.get_clock_info('monotonic').resolution + self._exception_handler = None + self.set_debug((not sys.flags.ignore_environment + and bool(os.environ.get('PYTHONASYNCIODEBUG')))) + # In debug mode, if the execution of a callback or a step of a task + # exceed this duration in seconds, the slow callback/task is logged. + self.slow_callback_duration = 0.1 + self._current_handle = None + self._task_factory = None + self._coroutine_wrapper_set = False + + if hasattr(sys, 'get_asyncgen_hooks'): + # Python >= 3.6 + # A weak set of all asynchronous generators that are + # being iterated by the loop. + self._asyncgens = weakref.WeakSet() + else: + self._asyncgens = None + + # Set to True when `loop.shutdown_asyncgens` is called. + self._asyncgens_shutdown_called = False + + def __repr__(self): + return ('<%s running=%s closed=%s debug=%s>' + % (self.__class__.__name__, self.is_running(), + self.is_closed(), self.get_debug())) + + def create_future(self): + """Create a Future object attached to the loop.""" + return futures.Future(loop=self) + + def create_task(self, coro): + """Schedule a coroutine object. + + Return a task object. + """ + self._check_closed() + if self._task_factory is None: + task = tasks.Task(coro, loop=self) + if task._source_traceback: + del task._source_traceback[-1] + else: + task = self._task_factory(self, coro) + return task + + def set_task_factory(self, factory): + """Set a task factory that will be used by loop.create_task(). + + If factory is None the default task factory will be set. + + If factory is a callable, it should have a signature matching + '(loop, coro)', where 'loop' will be a reference to the active + event loop, 'coro' will be a coroutine object. The callable + must return a Future. + """ + if factory is not None and not callable(factory): + raise TypeError('task factory must be a callable or None') + self._task_factory = factory + + def get_task_factory(self): + """Return a task factory, or None if the default one is in use.""" + return self._task_factory + + def _make_socket_transport(self, sock, protocol, waiter=None, *, + extra=None, server=None): + """Create socket transport.""" + raise NotImplementedError + + def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None): + """Create SSL transport.""" + raise NotImplementedError + + def _make_datagram_transport(self, sock, protocol, + address=None, waiter=None, extra=None): + """Create datagram transport.""" + raise NotImplementedError + + def _make_read_pipe_transport(self, pipe, protocol, waiter=None, + extra=None): + """Create read pipe transport.""" + raise NotImplementedError + + def _make_write_pipe_transport(self, pipe, protocol, waiter=None, + extra=None): + """Create write pipe transport.""" + raise NotImplementedError + + @coroutine + def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): + """Create subprocess transport.""" + raise NotImplementedError + + def _write_to_self(self): + """Write a byte to self-pipe, to wake up the event loop. + + This may be called from a different thread. + + The subclass is responsible for implementing the self-pipe. + """ + raise NotImplementedError + + def _process_events(self, event_list): + """Process selector events.""" + raise NotImplementedError + + def _check_closed(self): + if self._closed: + raise RuntimeError('Event loop is closed') + + def _asyncgen_finalizer_hook(self, agen): + self._asyncgens.discard(agen) + if not self.is_closed(): + self.create_task(agen.aclose()) + # Wake up the loop if the finalizer was called from + # a different thread. + self._write_to_self() + + def _asyncgen_firstiter_hook(self, agen): + if self._asyncgens_shutdown_called: + warnings.warn( + "asynchronous generator {!r} was scheduled after " + "loop.shutdown_asyncgens() call".format(agen), + ResourceWarning, source=self) + + self._asyncgens.add(agen) + + @coroutine + def shutdown_asyncgens(self): + """Shutdown all active asynchronous generators.""" + self._asyncgens_shutdown_called = True + + if self._asyncgens is None or not len(self._asyncgens): + # If Python version is <3.6 or we don't have any asynchronous + # generators alive. + return + + closing_agens = list(self._asyncgens) + self._asyncgens.clear() + + shutdown_coro = tasks.gather( + *[ag.aclose() for ag in closing_agens], + return_exceptions=True, + loop=self) + + results = yield from shutdown_coro + for result, agen in zip(results, closing_agens): + if isinstance(result, Exception): + self.call_exception_handler({ + 'message': 'an error occurred during closing of ' + 'asynchronous generator {!r}'.format(agen), + 'exception': result, + 'asyncgen': agen + }) + + def run_forever(self): + """Run until stop() is called.""" + self._check_closed() + if self.is_running(): + raise RuntimeError('This event loop is already running') + if events._get_running_loop() is not None: + raise RuntimeError( + 'Cannot run the event loop while another loop is running') + self._set_coroutine_wrapper(self._debug) + self._thread_id = threading.get_ident() + if self._asyncgens is not None: + old_agen_hooks = sys.get_asyncgen_hooks() + sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook) + try: + events._set_running_loop(self) + while True: + self._run_once() + if self._stopping: + break + finally: + self._stopping = False + self._thread_id = None + events._set_running_loop(None) + self._set_coroutine_wrapper(False) + if self._asyncgens is not None: + sys.set_asyncgen_hooks(*old_agen_hooks) + + def run_until_complete(self, future): + """Run until the Future is done. + + If the argument is a coroutine, it is wrapped in a Task. + + WARNING: It would be disastrous to call run_until_complete() + with the same coroutine twice -- it would wrap it in two + different Tasks and that can't be good. + + Return the Future's result, or raise its exception. + """ + self._check_closed() + + new_task = not futures.isfuture(future) + future = tasks.ensure_future(future, loop=self) + if new_task: + # An exception is raised if the future didn't complete, so there + # is no need to log the "destroy pending task" message + future._log_destroy_pending = False + + future.add_done_callback(_run_until_complete_cb) + try: + self.run_forever() + except: + if new_task and future.done() and not future.cancelled(): + # The coroutine raised a BaseException. Consume the exception + # to not log a warning, the caller doesn't have access to the + # local task. + future.exception() + raise + future.remove_done_callback(_run_until_complete_cb) + if not future.done(): + raise RuntimeError('Event loop stopped before Future completed.') + + return future.result() + + def stop(self): + """Stop running the event loop. + + Every callback already scheduled will still run. This simply informs + run_forever to stop looping after a complete iteration. + """ + self._stopping = True + + def close(self): + """Close the event loop. + + This clears the queues and shuts down the executor, + but does not wait for the executor to finish. + + The event loop must not be running. + """ + if self.is_running(): + raise RuntimeError("Cannot close a running event loop") + if self._closed: + return + if self._debug: + logger.debug("Close %r", self) + self._closed = True + self._ready.clear() + self._scheduled.clear() + executor = self._default_executor + if executor is not None: + self._default_executor = None + executor.shutdown(wait=False) + + def is_closed(self): + """Returns True if the event loop was closed.""" + return self._closed + + # On Python 3.3 and older, objects with a destructor part of a reference + # cycle are never destroyed. It's not more the case on Python 3.4 thanks + # to the PEP 442. + if compat.PY34: + def __del__(self): + if not self.is_closed(): + warnings.warn("unclosed event loop %r" % self, ResourceWarning) + if not self.is_running(): + self.close() + + def is_running(self): + """Returns True if the event loop is running.""" + return (self._thread_id is not None) + + def time(self): + """Return the time according to the event loop's clock. + + This is a float expressed in seconds since an epoch, but the + epoch, precision, accuracy and drift are unspecified and may + differ per event loop. + """ + return time.monotonic() + + def call_later(self, delay, callback, *args): + """Arrange for a callback to be called at a given time. + + Return a Handle: an opaque object with a cancel() method that + can be used to cancel the call. + + The delay can be an int or float, expressed in seconds. It is + always relative to the current time. + + Each callback will be called exactly once. If two callbacks + are scheduled for exactly the same time, it undefined which + will be called first. + + Any positional arguments after the callback will be passed to + the callback when it is called. + """ + timer = self.call_at(self.time() + delay, callback, *args) + if timer._source_traceback: + del timer._source_traceback[-1] + return timer + + def call_at(self, when, callback, *args): + """Like call_later(), but uses an absolute time. + + Absolute time corresponds to the event loop's time() method. + """ + self._check_closed() + if self._debug: + self._check_thread() + self._check_callback(callback, 'call_at') + timer = events.TimerHandle(when, callback, args, self) + if timer._source_traceback: + del timer._source_traceback[-1] + heapq.heappush(self._scheduled, timer) + timer._scheduled = True + return timer + + def call_soon(self, callback, *args): + """Arrange for a callback to be called as soon as possible. + + This operates as a FIFO queue: callbacks are called in the + order in which they are registered. Each callback will be + called exactly once. + + Any positional arguments after the callback will be passed to + the callback when it is called. + """ + self._check_closed() + if self._debug: + self._check_thread() + self._check_callback(callback, 'call_soon') + handle = self._call_soon(callback, args) + if handle._source_traceback: + del handle._source_traceback[-1] + return handle + + def _check_callback(self, callback, method): + if (coroutines.iscoroutine(callback) or + coroutines.iscoroutinefunction(callback)): + raise TypeError( + "coroutines cannot be used with {}()".format(method)) + if not callable(callback): + raise TypeError( + 'a callable object was expected by {}(), got {!r}'.format( + method, callback)) + + + def _call_soon(self, callback, args): + handle = events.Handle(callback, args, self) + if handle._source_traceback: + del handle._source_traceback[-1] + self._ready.append(handle) + return handle + + def _check_thread(self): + """Check that the current thread is the thread running the event loop. + + Non-thread-safe methods of this class make this assumption and will + likely behave incorrectly when the assumption is violated. + + Should only be called when (self._debug == True). The caller is + responsible for checking this condition for performance reasons. + """ + if self._thread_id is None: + return + thread_id = threading.get_ident() + if thread_id != self._thread_id: + raise RuntimeError( + "Non-thread-safe operation invoked on an event loop other " + "than the current one") + + def call_soon_threadsafe(self, callback, *args): + """Like call_soon(), but thread-safe.""" + self._check_closed() + if self._debug: + self._check_callback(callback, 'call_soon_threadsafe') + handle = self._call_soon(callback, args) + if handle._source_traceback: + del handle._source_traceback[-1] + self._write_to_self() + return handle + + def run_in_executor(self, executor, func, *args): + self._check_closed() + if self._debug: + self._check_callback(func, 'run_in_executor') + if executor is None: + executor = self._default_executor + if executor is None: + try: + executor = concurrent.futures.ThreadPoolExecutor() + except: + executor = concurrent.futures.ThreadPoolExecutor(_MAX_WORKERS) + self._default_executor = executor + return futures.wrap_future(executor.submit(func, *args), loop=self) + + def set_default_executor(self, executor): + self._default_executor = executor + + def _getaddrinfo_debug(self, host, port, family, type, proto, flags): + msg = ["%s:%r" % (host, port)] + if family: + msg.append('family=%r' % family) + if type: + msg.append('type=%r' % type) + if proto: + msg.append('proto=%r' % proto) + if flags: + msg.append('flags=%r' % flags) + msg = ', '.join(msg) + logger.debug('Get address info %s', msg) + + t0 = self.time() + addrinfo = socket.getaddrinfo(host, port, family, type, proto, flags) + dt = self.time() - t0 + + msg = ('Getting address info %s took %.3f ms: %r' + % (msg, dt * 1e3, addrinfo)) + if dt >= self.slow_callback_duration: + logger.info(msg) + else: + logger.debug(msg) + return addrinfo + + def getaddrinfo(self, host, port, *, + family=0, type=0, proto=0, flags=0): + if self._debug: + return self.run_in_executor(None, self._getaddrinfo_debug, + host, port, family, type, proto, flags) + else: + return self.run_in_executor(None, socket.getaddrinfo, + host, port, family, type, proto, flags) + + def getnameinfo(self, sockaddr, flags=0): + return self.run_in_executor(None, socket.getnameinfo, sockaddr, flags) + + @coroutine + def create_connection(self, protocol_factory, host=None, port=None, *, + ssl=None, family=0, proto=0, flags=0, sock=None, + local_addr=None, server_hostname=None): + """Connect to a TCP server. + + Create a streaming transport connection to a given Internet host and + port: socket family AF_INET or socket.AF_INET6 depending on host (or + family if specified), socket type SOCK_STREAM. protocol_factory must be + a callable returning a protocol instance. + + This method is a coroutine which will try to establish the connection + in the background. When successful, the coroutine returns a + (transport, protocol) pair. + """ + if server_hostname is not None and not ssl: + raise ValueError('server_hostname is only meaningful with ssl') + + if server_hostname is None and ssl: + # Use host as default for server_hostname. It is an error + # if host is empty or not set, e.g. when an + # already-connected socket was passed or when only a port + # is given. To avoid this error, you can pass + # server_hostname='' -- this will bypass the hostname + # check. (This also means that if host is a numeric + # IP/IPv6 address, we will attempt to verify that exact + # address; this will probably fail, but it is possible to + # create a certificate for a specific IP address, so we + # don't judge it here.) + if not host: + raise ValueError('You must set server_hostname ' + 'when using ssl without a host') + server_hostname = host + + if host is not None or port is not None: + if sock is not None: + raise ValueError( + 'host/port and sock can not be specified at the same time') + + f1 = _ensure_resolved((host, port), family=family, + type=socket.SOCK_STREAM, proto=proto, + flags=flags, loop=self) + fs = [f1] + if local_addr is not None: + f2 = _ensure_resolved(local_addr, family=family, + type=socket.SOCK_STREAM, proto=proto, + flags=flags, loop=self) + fs.append(f2) + else: + f2 = None + + yield from tasks.wait(fs, loop=self) + + infos = f1.result() + if not infos: + raise OSError('getaddrinfo() returned empty list') + if f2 is not None: + laddr_infos = f2.result() + if not laddr_infos: + raise OSError('getaddrinfo() returned empty list') + + exceptions = [] + for family, type, proto, cname, address in infos: + try: + sock = socket.socket(family=family, type=type, proto=proto) + sock.setblocking(False) + if f2 is not None: + for _, _, _, _, laddr in laddr_infos: + try: + sock.bind(laddr) + break + except OSError as exc: + exc = OSError( + exc.errno, 'error while ' + 'attempting to bind on address ' + '{!r}: {}'.format( + laddr, exc.strerror.lower())) + exceptions.append(exc) + else: + sock.close() + sock = None + continue + if self._debug: + logger.debug("connect %r to %r", sock, address) + yield from self.sock_connect(sock, address) + except OSError as exc: + if sock is not None: + sock.close() + exceptions.append(exc) + except: + if sock is not None: + sock.close() + raise + else: + break + else: + if len(exceptions) == 1: + raise exceptions[0] + else: + # If they all have the same str(), raise one. + model = str(exceptions[0]) + if all(str(exc) == model for exc in exceptions): + raise exceptions[0] + # Raise a combined exception so the user can see all + # the various error messages. + raise OSError('Multiple exceptions: {}'.format( + ', '.join(str(exc) for exc in exceptions))) + + elif sock is None: + raise ValueError( + 'host and port was not specified and no sock specified') + + transport, protocol = yield from self._create_connection_transport( + sock, protocol_factory, ssl, server_hostname) + if self._debug: + # Get the socket from the transport because SSL transport closes + # the old socket and creates a new SSL socket + sock = transport.get_extra_info('socket') + logger.debug("%r connected to %s:%r: (%r, %r)", + sock, host, port, transport, protocol) + return transport, protocol + + @coroutine + def _create_connection_transport(self, sock, protocol_factory, ssl, + server_hostname, server_side=False): + + sock.setblocking(False) + + protocol = protocol_factory() + waiter = self.create_future() + if ssl: + sslcontext = None if isinstance(ssl, bool) else ssl + transport = self._make_ssl_transport( + sock, protocol, sslcontext, waiter, + server_side=server_side, server_hostname=server_hostname) + else: + transport = self._make_socket_transport(sock, protocol, waiter) + + try: + yield from waiter + except: + transport.close() + raise + + return transport, protocol + + @coroutine + def create_datagram_endpoint(self, protocol_factory, + local_addr=None, remote_addr=None, *, + family=0, proto=0, flags=0, + reuse_address=None, reuse_port=None, + allow_broadcast=None, sock=None): + """Create datagram connection.""" + if sock is not None: + if (local_addr or remote_addr or + family or proto or flags or + reuse_address or reuse_port or allow_broadcast): + # show the problematic kwargs in exception msg + opts = dict(local_addr=local_addr, remote_addr=remote_addr, + family=family, proto=proto, flags=flags, + reuse_address=reuse_address, reuse_port=reuse_port, + allow_broadcast=allow_broadcast) + problems = ', '.join( + '{}={}'.format(k, v) for k, v in opts.items() if v) + raise ValueError( + 'socket modifier keyword arguments can not be used ' + 'when sock is specified. ({})'.format(problems)) + sock.setblocking(False) + r_addr = None + else: + if not (local_addr or remote_addr): + if family == 0: + raise ValueError('unexpected address family') + addr_pairs_info = (((family, proto), (None, None)),) + else: + # join address by (family, protocol) + addr_infos = collections.OrderedDict() + for idx, addr in ((0, local_addr), (1, remote_addr)): + if addr is not None: + assert isinstance(addr, tuple) and len(addr) == 2, ( + '2-tuple is expected') + + infos = yield from _ensure_resolved( + addr, family=family, type=socket.SOCK_DGRAM, + proto=proto, flags=flags, loop=self) + if not infos: + raise OSError('getaddrinfo() returned empty list') + + for fam, _, pro, _, address in infos: + key = (fam, pro) + if key not in addr_infos: + addr_infos[key] = [None, None] + addr_infos[key][idx] = address + + # each addr has to have info for each (family, proto) pair + addr_pairs_info = [ + (key, addr_pair) for key, addr_pair in addr_infos.items() + if not ((local_addr and addr_pair[0] is None) or + (remote_addr and addr_pair[1] is None))] + + if not addr_pairs_info: + raise ValueError('can not get address information') + + exceptions = [] + + if reuse_address is None: + reuse_address = os.name == 'posix' and sys.platform != 'cygwin' + + for ((family, proto), + (local_address, remote_address)) in addr_pairs_info: + sock = None + r_addr = None + try: + sock = socket.socket( + family=family, type=socket.SOCK_DGRAM, proto=proto) + if reuse_address: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if reuse_port: + _set_reuseport(sock) + if allow_broadcast: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + sock.setblocking(False) + + if local_addr: + sock.bind(local_address) + if remote_addr: + yield from self.sock_connect(sock, remote_address) + r_addr = remote_address + except OSError as exc: + if sock is not None: + sock.close() + exceptions.append(exc) + except: + if sock is not None: + sock.close() + raise + else: + break + else: + raise exceptions[0] + + protocol = protocol_factory() + waiter = self.create_future() + transport = self._make_datagram_transport( + sock, protocol, r_addr, waiter) + if self._debug: + if local_addr: + logger.info("Datagram endpoint local_addr=%r remote_addr=%r " + "created: (%r, %r)", + local_addr, remote_addr, transport, protocol) + else: + logger.debug("Datagram endpoint remote_addr=%r created: " + "(%r, %r)", + remote_addr, transport, protocol) + + try: + yield from waiter + except: + transport.close() + raise + + return transport, protocol + + @coroutine + def _create_server_getaddrinfo(self, host, port, family, flags): + infos = yield from _ensure_resolved((host, port), family=family, + type=socket.SOCK_STREAM, + flags=flags, loop=self) + if not infos: + raise OSError('getaddrinfo({!r}) returned empty list'.format(host)) + return infos + + @coroutine + def create_server(self, protocol_factory, host=None, port=None, + *, + family=socket.AF_UNSPEC, + flags=socket.AI_PASSIVE, + sock=None, + backlog=100, + ssl=None, + reuse_address=None, + reuse_port=None): + """Create a TCP server. + + The host parameter can be a string, in that case the TCP server is bound + to host and port. + + The host parameter can also be a sequence of strings and in that case + the TCP server is bound to all hosts of the sequence. If a host + appears multiple times (possibly indirectly e.g. when hostnames + resolve to the same IP address), the server is only bound once to that + host. + + Return a Server object which can be used to stop the service. + + This method is a coroutine. + """ + if isinstance(ssl, bool): + raise TypeError('ssl argument must be an SSLContext or None') + if host is not None or port is not None: + if sock is not None: + raise ValueError( + 'host/port and sock can not be specified at the same time') + + AF_INET6 = getattr(socket, 'AF_INET6', 0) + if reuse_address is None: + reuse_address = os.name == 'posix' and sys.platform != 'cygwin' + sockets = [] + if host == '': + hosts = [None] + elif (isinstance(host, str) or + not isinstance(host, collections.Iterable)): + hosts = [host] + else: + hosts = host + + fs = [self._create_server_getaddrinfo(host, port, family=family, + flags=flags) + for host in hosts] + infos = yield from tasks.gather(*fs, loop=self) + infos = set(itertools.chain.from_iterable(infos)) + + completed = False + try: + for res in infos: + af, socktype, proto, canonname, sa = res + try: + sock = socket.socket(af, socktype, proto) + except socket.error: + # Assume it's a bad family/type/protocol combination. + if self._debug: + logger.warning('create_server() failed to create ' + 'socket.socket(%r, %r, %r)', + af, socktype, proto, exc_info=True) + continue + sockets.append(sock) + if reuse_address: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + if reuse_port: + _set_reuseport(sock) + # Disable IPv4/IPv6 dual stack support (enabled by + # default on Linux) which makes a single socket + # listen on both address families. + if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'): + sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_V6ONLY, + True) + try: + sock.bind(sa) + except OSError as err: + raise OSError(err.errno, 'error while attempting ' + 'to bind on address %r: %s' + % (sa, err.strerror.lower())) + completed = True + finally: + if not completed: + for sock in sockets: + sock.close() + else: + if sock is None: + raise ValueError('Neither host/port nor sock were specified') + sockets = [sock] + + server = Server(self, sockets) + for sock in sockets: + sock.listen(backlog) + sock.setblocking(False) + self._start_serving(protocol_factory, sock, ssl, server, backlog) + if self._debug: + logger.info("%r is serving", server) + return server + + @coroutine + def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None): + """Handle an accepted connection. + + This is used by servers that accept connections outside of + asyncio but that use asyncio to handle connections. + + This method is a coroutine. When completed, the coroutine + returns a (transport, protocol) pair. + """ + transport, protocol = yield from self._create_connection_transport( + sock, protocol_factory, ssl, '', server_side=True) + if self._debug: + # Get the socket from the transport because SSL transport closes + # the old socket and creates a new SSL socket + sock = transport.get_extra_info('socket') + logger.debug("%r handled: (%r, %r)", sock, transport, protocol) + return transport, protocol + + @coroutine + def connect_read_pipe(self, protocol_factory, pipe): + protocol = protocol_factory() + waiter = self.create_future() + transport = self._make_read_pipe_transport(pipe, protocol, waiter) + + try: + yield from waiter + except: + transport.close() + raise + + if self._debug: + logger.debug('Read pipe %r connected: (%r, %r)', + pipe.fileno(), transport, protocol) + return transport, protocol + + @coroutine + def connect_write_pipe(self, protocol_factory, pipe): + protocol = protocol_factory() + waiter = self.create_future() + transport = self._make_write_pipe_transport(pipe, protocol, waiter) + + try: + yield from waiter + except: + transport.close() + raise + + if self._debug: + logger.debug('Write pipe %r connected: (%r, %r)', + pipe.fileno(), transport, protocol) + return transport, protocol + + def _log_subprocess(self, msg, stdin, stdout, stderr): + info = [msg] + if stdin is not None: + info.append('stdin=%s' % _format_pipe(stdin)) + if stdout is not None and stderr == subprocess.STDOUT: + info.append('stdout=stderr=%s' % _format_pipe(stdout)) + else: + if stdout is not None: + info.append('stdout=%s' % _format_pipe(stdout)) + if stderr is not None: + info.append('stderr=%s' % _format_pipe(stderr)) + logger.debug(' '.join(info)) + + @coroutine + def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=False, shell=True, bufsize=0, + **kwargs): + if not isinstance(cmd, (bytes, str)): + raise ValueError("cmd must be a string") + if universal_newlines: + raise ValueError("universal_newlines must be False") + if not shell: + raise ValueError("shell must be True") + if bufsize != 0: + raise ValueError("bufsize must be 0") + protocol = protocol_factory() + if self._debug: + # don't log parameters: they may contain sensitive information + # (password) and may be too long + debug_log = 'run shell command %r' % cmd + self._log_subprocess(debug_log, stdin, stdout, stderr) + transport = yield from self._make_subprocess_transport( + protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs) + if self._debug: + logger.info('%s: %r', debug_log, transport) + return transport, protocol + + @coroutine + def subprocess_exec(self, protocol_factory, program, *args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=False, + shell=False, bufsize=0, **kwargs): + if universal_newlines: + raise ValueError("universal_newlines must be False") + if shell: + raise ValueError("shell must be False") + if bufsize != 0: + raise ValueError("bufsize must be 0") + popen_args = (program,) + args + for arg in popen_args: + if not isinstance(arg, (str, bytes)): + raise TypeError("program arguments must be " + "a bytes or text string, not %s" + % type(arg).__name__) + protocol = protocol_factory() + if self._debug: + # don't log parameters: they may contain sensitive information + # (password) and may be too long + debug_log = 'execute program %r' % program + self._log_subprocess(debug_log, stdin, stdout, stderr) + transport = yield from self._make_subprocess_transport( + protocol, popen_args, False, stdin, stdout, stderr, + bufsize, **kwargs) + if self._debug: + logger.info('%s: %r', debug_log, transport) + return transport, protocol + + def get_exception_handler(self): + """Return an exception handler, or None if the default one is in use. + """ + return self._exception_handler + + def set_exception_handler(self, handler): + """Set handler as the new event loop exception handler. + + If handler is None, the default exception handler will + be set. + + If handler is a callable object, it should have a + signature matching '(loop, context)', where 'loop' + will be a reference to the active event loop, 'context' + will be a dict object (see `call_exception_handler()` + documentation for details about context). + """ + if handler is not None and not callable(handler): + raise TypeError('A callable object or None is expected, ' + 'got {!r}'.format(handler)) + self._exception_handler = handler + + def default_exception_handler(self, context): + """Default exception handler. + + This is called when an exception occurs and no exception + handler is set, and can be called by a custom exception + handler that wants to defer to the default behavior. + + The context parameter has the same meaning as in + `call_exception_handler()`. + """ + message = context.get('message') + if not message: + message = 'Unhandled exception in event loop' + + exception = context.get('exception') + if exception is not None: + exc_info = (type(exception), exception, exception.__traceback__) + else: + exc_info = False + + if ('source_traceback' not in context + and self._current_handle is not None + and self._current_handle._source_traceback): + context['handle_traceback'] = self._current_handle._source_traceback + + log_lines = [message] + for key in sorted(context): + if key in {'message', 'exception'}: + continue + value = context[key] + if key == 'source_traceback': + tb = ''.join(traceback.format_list(value)) + value = 'Object created at (most recent call last):\n' + value += tb.rstrip() + elif key == 'handle_traceback': + tb = ''.join(traceback.format_list(value)) + value = 'Handle created at (most recent call last):\n' + value += tb.rstrip() + else: + value = repr(value) + log_lines.append('{}: {}'.format(key, value)) + + logger.error('\n'.join(log_lines), exc_info=exc_info) + + def call_exception_handler(self, context): + """Call the current event loop's exception handler. + + The context argument is a dict containing the following keys: + + - 'message': Error message; + - 'exception' (optional): Exception object; + - 'future' (optional): Future instance; + - 'handle' (optional): Handle instance; + - 'protocol' (optional): Protocol instance; + - 'transport' (optional): Transport instance; + - 'socket' (optional): Socket instance; + - 'asyncgen' (optional): Asynchronous generator that caused + the exception. + + New keys maybe introduced in the future. + + Note: do not overload this method in an event loop subclass. + For custom exception handling, use the + `set_exception_handler()` method. + """ + if self._exception_handler is None: + try: + self.default_exception_handler(context) + except Exception: + # Second protection layer for unexpected errors + # in the default implementation, as well as for subclassed + # event loops with overloaded "default_exception_handler". + logger.error('Exception in default exception handler', + exc_info=True) + else: + try: + self._exception_handler(self, context) + except Exception as exc: + # Exception in the user set custom exception handler. + try: + # Let's try default handler. + self.default_exception_handler({ + 'message': 'Unhandled error in exception handler', + 'exception': exc, + 'context': context, + }) + except Exception: + # Guard 'default_exception_handler' in case it is + # overloaded. + logger.error('Exception in default exception handler ' + 'while handling an unexpected error ' + 'in custom exception handler', + exc_info=True) + + def _add_callback(self, handle): + """Add a Handle to _scheduled (TimerHandle) or _ready.""" + assert isinstance(handle, events.Handle), 'A Handle is required here' + if handle._cancelled: + return + assert not isinstance(handle, events.TimerHandle) + self._ready.append(handle) + + def _add_callback_signalsafe(self, handle): + """Like _add_callback() but called from a signal handler.""" + self._add_callback(handle) + self._write_to_self() + + def _timer_handle_cancelled(self, handle): + """Notification that a TimerHandle has been cancelled.""" + if handle._scheduled: + self._timer_cancelled_count += 1 + + def _run_once(self): + """Run one full iteration of the event loop. + + This calls all currently ready callbacks, polls for I/O, + schedules the resulting callbacks, and finally schedules + 'call_later' callbacks. + """ + + sched_count = len(self._scheduled) + if (sched_count > _MIN_SCHEDULED_TIMER_HANDLES and + self._timer_cancelled_count / sched_count > + _MIN_CANCELLED_TIMER_HANDLES_FRACTION): + # Remove delayed calls that were cancelled if their number + # is too high + new_scheduled = [] + for handle in self._scheduled: + if handle._cancelled: + handle._scheduled = False + else: + new_scheduled.append(handle) + + heapq.heapify(new_scheduled) + self._scheduled = new_scheduled + self._timer_cancelled_count = 0 + else: + # Remove delayed calls that were cancelled from head of queue. + while self._scheduled and self._scheduled[0]._cancelled: + self._timer_cancelled_count -= 1 + handle = heapq.heappop(self._scheduled) + handle._scheduled = False + + timeout = None + if self._ready or self._stopping: + timeout = 0 + elif self._scheduled: + # Compute the desired timeout. + when = self._scheduled[0]._when + timeout = max(0, when - self.time()) + + if self._debug and timeout != 0: + t0 = self.time() + event_list = self._selector.select(timeout) + dt = self.time() - t0 + if dt >= 1.0: + level = logging.INFO + else: + level = logging.DEBUG + nevent = len(event_list) + if timeout is None: + logger.log(level, 'poll took %.3f ms: %s events', + dt * 1e3, nevent) + elif nevent: + logger.log(level, + 'poll %.3f ms took %.3f ms: %s events', + timeout * 1e3, dt * 1e3, nevent) + elif dt >= 1.0: + logger.log(level, + 'poll %.3f ms took %.3f ms: timeout', + timeout * 1e3, dt * 1e3) + else: + event_list = self._selector.select(timeout) + self._process_events(event_list) + + # Handle 'later' callbacks that are ready. + end_time = self.time() + self._clock_resolution + while self._scheduled: + handle = self._scheduled[0] + if handle._when >= end_time: + break + handle = heapq.heappop(self._scheduled) + handle._scheduled = False + self._ready.append(handle) + + # This is the only place where callbacks are actually *called*. + # All other places just add them to ready. + # Note: We run all currently scheduled callbacks, but not any + # callbacks scheduled by callbacks run this time around -- + # they will be run the next time (after another I/O poll). + # Use an idiom that is thread-safe without using locks. + ntodo = len(self._ready) + for i in range(ntodo): + handle = self._ready.popleft() + if handle._cancelled: + continue + if self._debug: + try: + self._current_handle = handle + t0 = self.time() + handle._run() + dt = self.time() - t0 + if dt >= self.slow_callback_duration: + logger.warning('Executing %s took %.3f seconds', + _format_handle(handle), dt) + finally: + self._current_handle = None + else: + handle._run() + handle = None # Needed to break cycles when an exception occurs. + + def _set_coroutine_wrapper(self, enabled): + try: + set_wrapper = sys.set_coroutine_wrapper + get_wrapper = sys.get_coroutine_wrapper + except AttributeError: + return + + enabled = bool(enabled) + if self._coroutine_wrapper_set == enabled: + return + + wrapper = coroutines.debug_wrapper + current_wrapper = get_wrapper() + + if enabled: + if current_wrapper not in (None, wrapper): + warnings.warn( + "loop.set_debug(True): cannot set debug coroutine " + "wrapper; another wrapper is already set %r" % + current_wrapper, RuntimeWarning) + else: + set_wrapper(wrapper) + self._coroutine_wrapper_set = True + else: + if current_wrapper not in (None, wrapper): + warnings.warn( + "loop.set_debug(False): cannot unset debug coroutine " + "wrapper; another wrapper was set %r" % + current_wrapper, RuntimeWarning) + else: + set_wrapper(None) + self._coroutine_wrapper_set = False + + def get_debug(self): + return self._debug + + def set_debug(self, enabled): + self._debug = enabled + + if self.is_running(): + self._set_coroutine_wrapper(enabled) diff --git a/thirdparty/asyncio/asyncio/base_subprocess.py b/thirdparty/asyncio/asyncio/base_subprocess.py new file mode 100644 index 0000000..23742a1 --- /dev/null +++ b/thirdparty/asyncio/asyncio/base_subprocess.py @@ -0,0 +1,292 @@ +import collections +import subprocess +import warnings + +from . import compat +from . import protocols +from . import transports +from .coroutines import coroutine +from .log import logger + + +class BaseSubprocessTransport(transports.SubprocessTransport): + + def __init__(self, loop, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=None, extra=None, **kwargs): + super().__init__(extra) + self._closed = False + self._protocol = protocol + self._loop = loop + self._proc = None + self._pid = None + self._returncode = None + self._exit_waiters = [] + self._pending_calls = collections.deque() + self._pipes = {} + self._finished = False + + if stdin == subprocess.PIPE: + self._pipes[0] = None + if stdout == subprocess.PIPE: + self._pipes[1] = None + if stderr == subprocess.PIPE: + self._pipes[2] = None + + # Create the child process: set the _proc attribute + try: + self._start(args=args, shell=shell, stdin=stdin, stdout=stdout, + stderr=stderr, bufsize=bufsize, **kwargs) + except: + self.close() + raise + + self._pid = self._proc.pid + self._extra['subprocess'] = self._proc + + if self._loop.get_debug(): + if isinstance(args, (bytes, str)): + program = args + else: + program = args[0] + logger.debug('process %r created: pid %s', + program, self._pid) + + self._loop.create_task(self._connect_pipes(waiter)) + + def __repr__(self): + info = [self.__class__.__name__] + if self._closed: + info.append('closed') + if self._pid is not None: + info.append('pid=%s' % self._pid) + if self._returncode is not None: + info.append('returncode=%s' % self._returncode) + elif self._pid is not None: + info.append('running') + else: + info.append('not started') + + stdin = self._pipes.get(0) + if stdin is not None: + info.append('stdin=%s' % stdin.pipe) + + stdout = self._pipes.get(1) + stderr = self._pipes.get(2) + if stdout is not None and stderr is stdout: + info.append('stdout=stderr=%s' % stdout.pipe) + else: + if stdout is not None: + info.append('stdout=%s' % stdout.pipe) + if stderr is not None: + info.append('stderr=%s' % stderr.pipe) + + return '<%s>' % ' '.join(info) + + def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): + raise NotImplementedError + + def set_protocol(self, protocol): + self._protocol = protocol + + def get_protocol(self): + return self._protocol + + def is_closing(self): + return self._closed + + def close(self): + if self._closed: + return + self._closed = True + + for proto in self._pipes.values(): + if proto is None: + continue + proto.pipe.close() + + if (self._proc is not None + # the child process finished? + and self._returncode is None + # the child process finished but the transport was not notified yet? + and self._proc.poll() is None + ): + if self._loop.get_debug(): + logger.warning('Close running child process: kill %r', self) + + try: + self._proc.kill() + except ProcessLookupError: + pass + + # Don't clear the _proc reference yet: _post_init() may still run + + # On Python 3.3 and older, objects with a destructor part of a reference + # cycle are never destroyed. It's not more the case on Python 3.4 thanks + # to the PEP 442. + if compat.PY34: + def __del__(self): + if not self._closed: + warnings.warn("unclosed transport %r" % self, ResourceWarning) + self.close() + + def get_pid(self): + return self._pid + + def get_returncode(self): + return self._returncode + + def get_pipe_transport(self, fd): + if fd in self._pipes: + return self._pipes[fd].pipe + else: + return None + + def _check_proc(self): + if self._proc is None: + raise ProcessLookupError() + + def send_signal(self, signal): + self._check_proc() + self._proc.send_signal(signal) + + def terminate(self): + self._check_proc() + self._proc.terminate() + + def kill(self): + self._check_proc() + self._proc.kill() + + @coroutine + def _connect_pipes(self, waiter): + try: + proc = self._proc + loop = self._loop + + if proc.stdin is not None: + _, pipe = yield from loop.connect_write_pipe( + lambda: WriteSubprocessPipeProto(self, 0), + proc.stdin) + self._pipes[0] = pipe + + if proc.stdout is not None: + _, pipe = yield from loop.connect_read_pipe( + lambda: ReadSubprocessPipeProto(self, 1), + proc.stdout) + self._pipes[1] = pipe + + if proc.stderr is not None: + _, pipe = yield from loop.connect_read_pipe( + lambda: ReadSubprocessPipeProto(self, 2), + proc.stderr) + self._pipes[2] = pipe + + assert self._pending_calls is not None + + loop.call_soon(self._protocol.connection_made, self) + for callback, data in self._pending_calls: + loop.call_soon(callback, *data) + self._pending_calls = None + except Exception as exc: + if waiter is not None and not waiter.cancelled(): + waiter.set_exception(exc) + else: + if waiter is not None and not waiter.cancelled(): + waiter.set_result(None) + + def _call(self, cb, *data): + if self._pending_calls is not None: + self._pending_calls.append((cb, data)) + else: + self._loop.call_soon(cb, *data) + + def _pipe_connection_lost(self, fd, exc): + self._call(self._protocol.pipe_connection_lost, fd, exc) + self._try_finish() + + def _pipe_data_received(self, fd, data): + self._call(self._protocol.pipe_data_received, fd, data) + + def _process_exited(self, returncode): + assert returncode is not None, returncode + assert self._returncode is None, self._returncode + if self._loop.get_debug(): + logger.info('%r exited with return code %r', + self, returncode) + self._returncode = returncode + if self._proc.returncode is None: + # asyncio uses a child watcher: copy the status into the Popen + # object. On Python 3.6, it is required to avoid a ResourceWarning. + self._proc.returncode = returncode + self._call(self._protocol.process_exited) + self._try_finish() + + # wake up futures waiting for wait() + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(returncode) + self._exit_waiters = None + + @coroutine + def _wait(self): + """Wait until the process exit and return the process return code. + + This method is a coroutine.""" + if self._returncode is not None: + return self._returncode + + waiter = self._loop.create_future() + self._exit_waiters.append(waiter) + return (yield from waiter) + + def _try_finish(self): + assert not self._finished + if self._returncode is None: + return + if all(p is not None and p.disconnected + for p in self._pipes.values()): + self._finished = True + self._call(self._call_connection_lost, None) + + def _call_connection_lost(self, exc): + try: + self._protocol.connection_lost(exc) + finally: + self._loop = None + self._proc = None + self._protocol = None + + +class WriteSubprocessPipeProto(protocols.BaseProtocol): + + def __init__(self, proc, fd): + self.proc = proc + self.fd = fd + self.pipe = None + self.disconnected = False + + def connection_made(self, transport): + self.pipe = transport + + def __repr__(self): + return ('<%s fd=%s pipe=%r>' + % (self.__class__.__name__, self.fd, self.pipe)) + + def connection_lost(self, exc): + self.disconnected = True + self.proc._pipe_connection_lost(self.fd, exc) + self.proc = None + + def pause_writing(self): + self.proc._protocol.pause_writing() + + def resume_writing(self): + self.proc._protocol.resume_writing() + + +class ReadSubprocessPipeProto(WriteSubprocessPipeProto, + protocols.Protocol): + + def data_received(self, data): + self.proc._pipe_data_received(self.fd, data) diff --git a/thirdparty/asyncio/asyncio/compat.py b/thirdparty/asyncio/asyncio/compat.py new file mode 100644 index 0000000..4790bb4 --- /dev/null +++ b/thirdparty/asyncio/asyncio/compat.py @@ -0,0 +1,18 @@ +"""Compatibility helpers for the different Python versions.""" + +import sys + +PY34 = sys.version_info >= (3, 4) +PY35 = sys.version_info >= (3, 5) +PY352 = sys.version_info >= (3, 5, 2) + + +def flatten_list_bytes(list_of_data): + """Concatenate a sequence of bytes-like objects.""" + if not PY34: + # On Python 3.3 and older, bytes.join() doesn't handle + # memoryview. + list_of_data = ( + bytes(data) if isinstance(data, memoryview) else data + for data in list_of_data) + return b''.join(list_of_data) diff --git a/thirdparty/asyncio/asyncio/constants.py b/thirdparty/asyncio/asyncio/constants.py new file mode 100644 index 0000000..f9e1232 --- /dev/null +++ b/thirdparty/asyncio/asyncio/constants.py @@ -0,0 +1,7 @@ +"""Constants.""" + +# After the connection is lost, log warnings after this many write()s. +LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5 + +# Seconds to wait before retrying accept(). +ACCEPT_RETRY_DELAY = 1 diff --git a/thirdparty/asyncio/asyncio/coroutines.py b/thirdparty/asyncio/asyncio/coroutines.py new file mode 100644 index 0000000..1db7030 --- /dev/null +++ b/thirdparty/asyncio/asyncio/coroutines.py @@ -0,0 +1,328 @@ +__all__ = ['coroutine', + 'iscoroutinefunction', 'iscoroutine'] + +import functools +import inspect +import opcode +import os +import sys +import traceback +import types + +from . import compat +from . import events +from . import futures +from .log import logger + + +# Opcode of "yield from" instruction +_YIELD_FROM = opcode.opmap['YIELD_FROM'] + +# If you set _DEBUG to true, @coroutine will wrap the resulting +# generator objects in a CoroWrapper instance (defined below). That +# instance will log a message when the generator is never iterated +# over, which may happen when you forget to use "yield from" with a +# coroutine call. Note that the value of the _DEBUG flag is taken +# when the decorator is used, so to be of any use it must be set +# before you define your coroutines. A downside of using this feature +# is that tracebacks show entries for the CoroWrapper.__next__ method +# when _DEBUG is true. +_DEBUG = (not sys.flags.ignore_environment and + bool(os.environ.get('PYTHONASYNCIODEBUG'))) + + +try: + _types_coroutine = types.coroutine +except AttributeError: + _types_coroutine = None + +try: + _inspect_iscoroutinefunction = inspect.iscoroutinefunction +except AttributeError: + _inspect_iscoroutinefunction = lambda func: False + +try: + from collections.abc import Coroutine as _CoroutineABC, \ + Awaitable as _AwaitableABC +except ImportError: + _CoroutineABC = _AwaitableABC = None + + +# Check for CPython issue #21209 +def has_yield_from_bug(): + class MyGen: + def __init__(self): + self.send_args = None + def __iter__(self): + return self + def __next__(self): + return 42 + def send(self, *what): + self.send_args = what + return None + def yield_from_gen(gen): + yield from gen + value = (1, 2, 3) + gen = MyGen() + coro = yield_from_gen(gen) + next(coro) + coro.send(value) + return gen.send_args != (value,) +_YIELD_FROM_BUG = has_yield_from_bug() +del has_yield_from_bug + + +def debug_wrapper(gen): + # This function is called from 'sys.set_coroutine_wrapper'. + # We only wrap here coroutines defined via 'async def' syntax. + # Generator-based coroutines are wrapped in @coroutine + # decorator. + return CoroWrapper(gen, None) + + +class CoroWrapper: + # Wrapper for coroutine object in _DEBUG mode. + + def __init__(self, gen, func=None): + assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen + self.gen = gen + self.func = func # Used to unwrap @coroutine decorator + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + self.__name__ = getattr(gen, '__name__', None) + self.__qualname__ = getattr(gen, '__qualname__', None) + + def __repr__(self): + coro_repr = _format_coroutine(self) + if self._source_traceback: + frame = self._source_traceback[-1] + coro_repr += ', created at %s:%s' % (frame[0], frame[1]) + return '<%s %s>' % (self.__class__.__name__, coro_repr) + + def __iter__(self): + return self + + def __next__(self): + return self.gen.send(None) + + if _YIELD_FROM_BUG: + # For for CPython issue #21209: using "yield from" and a custom + # generator, generator.send(tuple) unpacks the tuple instead of passing + # the tuple unchanged. Check if the caller is a generator using "yield + # from" to decide if the parameter should be unpacked or not. + def send(self, *value): + frame = sys._getframe() + caller = frame.f_back + assert caller.f_lasti >= 0 + if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM: + value = value[0] + return self.gen.send(value) + else: + def send(self, value): + return self.gen.send(value) + + def throw(self, type, value=None, traceback=None): + return self.gen.throw(type, value, traceback) + + def close(self): + return self.gen.close() + + @property + def gi_frame(self): + return self.gen.gi_frame + + @property + def gi_running(self): + return self.gen.gi_running + + @property + def gi_code(self): + return self.gen.gi_code + + if compat.PY35: + + def __await__(self): + cr_await = getattr(self.gen, 'cr_await', None) + if cr_await is not None: + raise RuntimeError( + "Cannot await on coroutine {!r} while it's " + "awaiting for {!r}".format(self.gen, cr_await)) + return self + + @property + def gi_yieldfrom(self): + return self.gen.gi_yieldfrom + + @property + def cr_await(self): + return self.gen.cr_await + + @property + def cr_running(self): + return self.gen.cr_running + + @property + def cr_code(self): + return self.gen.cr_code + + @property + def cr_frame(self): + return self.gen.cr_frame + + def __del__(self): + # Be careful accessing self.gen.frame -- self.gen might not exist. + gen = getattr(self, 'gen', None) + frame = getattr(gen, 'gi_frame', None) + if frame is None: + frame = getattr(gen, 'cr_frame', None) + if frame is not None and frame.f_lasti == -1: + msg = '%r was never yielded from' % self + tb = getattr(self, '_source_traceback', ()) + if tb: + tb = ''.join(traceback.format_list(tb)) + msg += ('\nCoroutine object created at ' + '(most recent call last):\n') + msg += tb.rstrip() + logger.error(msg) + + +def coroutine(func): + """Decorator to mark coroutines. + + If the coroutine is not yielded from before it is destroyed, + an error message is logged. + """ + if _inspect_iscoroutinefunction(func): + # In Python 3.5 that's all we need to do for coroutines + # defiend with "async def". + # Wrapping in CoroWrapper will happen via + # 'sys.set_coroutine_wrapper' function. + return func + + if inspect.isgeneratorfunction(func): + coro = func + else: + @functools.wraps(func) + def coro(*args, **kw): + res = func(*args, **kw) + if (futures.isfuture(res) or inspect.isgenerator(res) or + isinstance(res, CoroWrapper)): + res = yield from res + elif _AwaitableABC is not None: + # If 'func' returns an Awaitable (new in 3.5) we + # want to run it. + try: + await_meth = res.__await__ + except AttributeError: + pass + else: + if isinstance(res, _AwaitableABC): + res = yield from await_meth() + return res + + if not _DEBUG: + if _types_coroutine is None: + wrapper = coro + else: + wrapper = _types_coroutine(coro) + else: + @functools.wraps(func) + def wrapper(*args, **kwds): + w = CoroWrapper(coro(*args, **kwds), func=func) + if w._source_traceback: + del w._source_traceback[-1] + # Python < 3.5 does not implement __qualname__ + # on generator objects, so we set it manually. + # We use getattr as some callables (such as + # functools.partial may lack __qualname__). + w.__name__ = getattr(func, '__name__', None) + w.__qualname__ = getattr(func, '__qualname__', None) + return w + + wrapper._is_coroutine = True # For iscoroutinefunction(). + return wrapper + + +def iscoroutinefunction(func): + """Return True if func is a decorated coroutine function.""" + return (getattr(func, '_is_coroutine', False) or + _inspect_iscoroutinefunction(func)) + + +_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper) +if _CoroutineABC is not None: + _COROUTINE_TYPES += (_CoroutineABC,) + + +def iscoroutine(obj): + """Return True if obj is a coroutine object.""" + return isinstance(obj, _COROUTINE_TYPES) + + +def _format_coroutine(coro): + assert iscoroutine(coro) + + if not hasattr(coro, 'cr_code') and not hasattr(coro, 'gi_code'): + # Most likely a Cython coroutine. + coro_name = getattr(coro, '__qualname__', coro.__name__) + coro_name = '{}()'.format(coro_name) + + running = False + try: + running = coro.cr_running + except AttributeError: + try: + running = coro.gi_running + except AttributeError: + pass + + if running: + return '{} running'.format(coro_name) + else: + return coro_name + + coro_name = None + if isinstance(coro, CoroWrapper): + func = coro.func + coro_name = coro.__qualname__ + if coro_name is not None: + coro_name = '{}()'.format(coro_name) + else: + func = coro + + if coro_name is None: + coro_name = events._format_callback(func, (), {}) + + try: + coro_code = coro.gi_code + except AttributeError: + coro_code = coro.cr_code + + try: + coro_frame = coro.gi_frame + except AttributeError: + coro_frame = coro.cr_frame + + filename = coro_code.co_filename + lineno = 0 + if (isinstance(coro, CoroWrapper) and + not inspect.isgeneratorfunction(coro.func) and + coro.func is not None): + source = events._get_function_source(coro.func) + if source is not None: + filename, lineno = source + if coro_frame is None: + coro_repr = ('%s done, defined at %s:%s' + % (coro_name, filename, lineno)) + else: + coro_repr = ('%s running, defined at %s:%s' + % (coro_name, filename, lineno)) + elif coro_frame is not None: + lineno = coro_frame.f_lineno + coro_repr = ('%s running at %s:%s' + % (coro_name, filename, lineno)) + else: + lineno = coro_code.co_firstlineno + coro_repr = ('%s done, defined at %s:%s' + % (coro_name, filename, lineno)) + + return coro_repr diff --git a/thirdparty/asyncio/asyncio/events.py b/thirdparty/asyncio/asyncio/events.py new file mode 100644 index 0000000..8575e2c --- /dev/null +++ b/thirdparty/asyncio/asyncio/events.py @@ -0,0 +1,691 @@ +"""Event loop and event loop policy.""" + +__all__ = ['AbstractEventLoopPolicy', + 'AbstractEventLoop', 'AbstractServer', + 'Handle', 'TimerHandle', + 'get_event_loop_policy', 'set_event_loop_policy', + 'get_event_loop', 'set_event_loop', 'new_event_loop', + 'get_child_watcher', 'set_child_watcher', + ] + +import functools +import inspect +import reprlib +import socket +import subprocess +import sys +import threading +import traceback + +from asyncio import compat + + +def _get_function_source(func): + if compat.PY34: + func = inspect.unwrap(func) + elif hasattr(func, '__wrapped__'): + func = func.__wrapped__ + if inspect.isfunction(func): + code = func.__code__ + return (code.co_filename, code.co_firstlineno) + if isinstance(func, functools.partial): + return _get_function_source(func.func) + if compat.PY34 and isinstance(func, functools.partialmethod): + return _get_function_source(func.func) + return None + + +def _format_args_and_kwargs(args, kwargs): + """Format function arguments and keyword arguments. + + Special case for a single parameter: ('hello',) is formatted as ('hello'). + """ + # use reprlib to limit the length of the output + items = [] + if args: + items.extend(reprlib.repr(arg) for arg in args) + if kwargs: + items.extend('{}={}'.format(k, reprlib.repr(v)) + for k, v in kwargs.items()) + return '(' + ', '.join(items) + ')' + + +def _format_callback(func, args, kwargs, suffix=''): + if isinstance(func, functools.partial): + suffix = _format_args_and_kwargs(args, kwargs) + suffix + return _format_callback(func.func, func.args, func.keywords, suffix) + + if hasattr(func, '__qualname__'): + func_repr = getattr(func, '__qualname__') + elif hasattr(func, '__name__'): + func_repr = getattr(func, '__name__') + else: + func_repr = repr(func) + + func_repr += _format_args_and_kwargs(args, kwargs) + if suffix: + func_repr += suffix + return func_repr + +def _format_callback_source(func, args): + func_repr = _format_callback(func, args, None) + source = _get_function_source(func) + if source: + func_repr += ' at %s:%s' % source + return func_repr + + +class Handle: + """Object returned by callback registration methods.""" + + __slots__ = ('_callback', '_args', '_cancelled', '_loop', + '_source_traceback', '_repr', '__weakref__') + + def __init__(self, callback, args, loop): + self._loop = loop + self._callback = callback + self._args = args + self._cancelled = False + self._repr = None + if self._loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + else: + self._source_traceback = None + + def _repr_info(self): + info = [self.__class__.__name__] + if self._cancelled: + info.append('cancelled') + if self._callback is not None: + info.append(_format_callback_source(self._callback, self._args)) + if self._source_traceback: + frame = self._source_traceback[-1] + info.append('created at %s:%s' % (frame[0], frame[1])) + return info + + def __repr__(self): + if self._repr is not None: + return self._repr + info = self._repr_info() + return '<%s>' % ' '.join(info) + + def cancel(self): + if not self._cancelled: + self._cancelled = True + if self._loop.get_debug(): + # Keep a representation in debug mode to keep callback and + # parameters. For example, to log the warning + # "Executing took 2.5 second" + self._repr = repr(self) + self._callback = None + self._args = None + + def _run(self): + try: + self._callback(*self._args) + except Exception as exc: + cb = _format_callback_source(self._callback, self._args) + msg = 'Exception in callback {}'.format(cb) + context = { + 'message': msg, + 'exception': exc, + 'handle': self, + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + self = None # Needed to break cycles when an exception occurs. + + +class TimerHandle(Handle): + """Object returned by timed callback registration methods.""" + + __slots__ = ['_scheduled', '_when'] + + def __init__(self, when, callback, args, loop): + assert when is not None + super().__init__(callback, args, loop) + if self._source_traceback: + del self._source_traceback[-1] + self._when = when + self._scheduled = False + + def _repr_info(self): + info = super()._repr_info() + pos = 2 if self._cancelled else 1 + info.insert(pos, 'when=%s' % self._when) + return info + + def __hash__(self): + return hash(self._when) + + def __lt__(self, other): + return self._when < other._when + + def __le__(self, other): + if self._when < other._when: + return True + return self.__eq__(other) + + def __gt__(self, other): + return self._when > other._when + + def __ge__(self, other): + if self._when > other._when: + return True + return self.__eq__(other) + + def __eq__(self, other): + if isinstance(other, TimerHandle): + return (self._when == other._when and + self._callback == other._callback and + self._args == other._args and + self._cancelled == other._cancelled) + return NotImplemented + + def __ne__(self, other): + equal = self.__eq__(other) + return NotImplemented if equal is NotImplemented else not equal + + def cancel(self): + if not self._cancelled: + self._loop._timer_handle_cancelled(self) + super().cancel() + + +class AbstractServer: + """Abstract server returned by create_server().""" + + def close(self): + """Stop serving. This leaves existing connections open.""" + return NotImplemented + + def wait_closed(self): + """Coroutine to wait until service is closed.""" + return NotImplemented + + +class AbstractEventLoop: + """Abstract event loop.""" + + # Running and stopping the event loop. + + def run_forever(self): + """Run the event loop until stop() is called.""" + raise NotImplementedError + + def run_until_complete(self, future): + """Run the event loop until a Future is done. + + Return the Future's result, or raise its exception. + """ + raise NotImplementedError + + def stop(self): + """Stop the event loop as soon as reasonable. + + Exactly how soon that is may depend on the implementation, but + no more I/O callbacks should be scheduled. + """ + raise NotImplementedError + + def is_running(self): + """Return whether the event loop is currently running.""" + raise NotImplementedError + + def is_closed(self): + """Returns True if the event loop was closed.""" + raise NotImplementedError + + def close(self): + """Close the loop. + + The loop should not be running. + + This is idempotent and irreversible. + + No other methods should be called after this one. + """ + raise NotImplementedError + + def shutdown_asyncgens(self): + """Shutdown all active asynchronous generators.""" + raise NotImplementedError + + # Methods scheduling callbacks. All these return Handles. + + def _timer_handle_cancelled(self, handle): + """Notification that a TimerHandle has been cancelled.""" + raise NotImplementedError + + def call_soon(self, callback, *args): + return self.call_later(0, callback, *args) + + def call_later(self, delay, callback, *args): + raise NotImplementedError + + def call_at(self, when, callback, *args): + raise NotImplementedError + + def time(self): + raise NotImplementedError + + def create_future(self): + raise NotImplementedError + + # Method scheduling a coroutine object: create a task. + + def create_task(self, coro): + raise NotImplementedError + + # Methods for interacting with threads. + + def call_soon_threadsafe(self, callback, *args): + raise NotImplementedError + + def run_in_executor(self, executor, func, *args): + raise NotImplementedError + + def set_default_executor(self, executor): + raise NotImplementedError + + # Network I/O methods returning Futures. + + def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0): + raise NotImplementedError + + def getnameinfo(self, sockaddr, flags=0): + raise NotImplementedError + + def create_connection(self, protocol_factory, host=None, port=None, *, + ssl=None, family=0, proto=0, flags=0, sock=None, + local_addr=None, server_hostname=None): + raise NotImplementedError + + def create_server(self, protocol_factory, host=None, port=None, *, + family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, + sock=None, backlog=100, ssl=None, reuse_address=None, + reuse_port=None): + """A coroutine which creates a TCP server bound to host and port. + + The return value is a Server object which can be used to stop + the service. + + If host is an empty string or None all interfaces are assumed + and a list of multiple sockets will be returned (most likely + one for IPv4 and another one for IPv6). The host parameter can also be a + sequence (e.g. list) of hosts to bind to. + + family can be set to either AF_INET or AF_INET6 to force the + socket to use IPv4 or IPv6. If not set it will be determined + from host (defaults to AF_UNSPEC). + + flags is a bitmask for getaddrinfo(). + + sock can optionally be specified in order to use a preexisting + socket object. + + backlog is the maximum number of queued connections passed to + listen() (defaults to 100). + + ssl can be set to an SSLContext to enable SSL over the + accepted connections. + + reuse_address tells the kernel to reuse a local socket in + TIME_WAIT state, without waiting for its natural timeout to + expire. If not specified will automatically be set to True on + UNIX. + + reuse_port tells the kernel to allow this endpoint to be bound to + the same port as other existing endpoints are bound to, so long as + they all set this flag when being created. This option is not + supported on Windows. + """ + raise NotImplementedError + + def create_unix_connection(self, protocol_factory, path, *, + ssl=None, sock=None, + server_hostname=None): + raise NotImplementedError + + def create_unix_server(self, protocol_factory, path, *, + sock=None, backlog=100, ssl=None): + """A coroutine which creates a UNIX Domain Socket server. + + The return value is a Server object, which can be used to stop + the service. + + path is a str, representing a file systsem path to bind the + server socket to. + + sock can optionally be specified in order to use a preexisting + socket object. + + backlog is the maximum number of queued connections passed to + listen() (defaults to 100). + + ssl can be set to an SSLContext to enable SSL over the + accepted connections. + """ + raise NotImplementedError + + def create_datagram_endpoint(self, protocol_factory, + local_addr=None, remote_addr=None, *, + family=0, proto=0, flags=0, + reuse_address=None, reuse_port=None, + allow_broadcast=None, sock=None): + """A coroutine which creates a datagram endpoint. + + This method will try to establish the endpoint in the background. + When successful, the coroutine returns a (transport, protocol) pair. + + protocol_factory must be a callable returning a protocol instance. + + socket family AF_INET or socket.AF_INET6 depending on host (or + family if specified), socket type SOCK_DGRAM. + + reuse_address tells the kernel to reuse a local socket in + TIME_WAIT state, without waiting for its natural timeout to + expire. If not specified it will automatically be set to True on + UNIX. + + reuse_port tells the kernel to allow this endpoint to be bound to + the same port as other existing endpoints are bound to, so long as + they all set this flag when being created. This option is not + supported on Windows and some UNIX's. If the + :py:data:`~socket.SO_REUSEPORT` constant is not defined then this + capability is unsupported. + + allow_broadcast tells the kernel to allow this endpoint to send + messages to the broadcast address. + + sock can optionally be specified in order to use a preexisting + socket object. + """ + raise NotImplementedError + + # Pipes and subprocesses. + + def connect_read_pipe(self, protocol_factory, pipe): + """Register read pipe in event loop. Set the pipe to non-blocking mode. + + protocol_factory should instantiate object with Protocol interface. + pipe is a file-like object. + Return pair (transport, protocol), where transport supports the + ReadTransport interface.""" + # The reason to accept file-like object instead of just file descriptor + # is: we need to own pipe and close it at transport finishing + # Can got complicated errors if pass f.fileno(), + # close fd in pipe transport then close f and vise versa. + raise NotImplementedError + + def connect_write_pipe(self, protocol_factory, pipe): + """Register write pipe in event loop. + + protocol_factory should instantiate object with BaseProtocol interface. + Pipe is file-like object already switched to nonblocking. + Return pair (transport, protocol), where transport support + WriteTransport interface.""" + # The reason to accept file-like object instead of just file descriptor + # is: we need to own pipe and close it at transport finishing + # Can got complicated errors if pass f.fileno(), + # close fd in pipe transport then close f and vise versa. + raise NotImplementedError + + def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + **kwargs): + raise NotImplementedError + + def subprocess_exec(self, protocol_factory, *args, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + **kwargs): + raise NotImplementedError + + # Ready-based callback registration methods. + # The add_*() methods return None. + # The remove_*() methods return True if something was removed, + # False if there was nothing to delete. + + def add_reader(self, fd, callback, *args): + raise NotImplementedError + + def remove_reader(self, fd): + raise NotImplementedError + + def add_writer(self, fd, callback, *args): + raise NotImplementedError + + def remove_writer(self, fd): + raise NotImplementedError + + # Completion based I/O methods returning Futures. + + def sock_recv(self, sock, nbytes): + raise NotImplementedError + + def sock_sendall(self, sock, data): + raise NotImplementedError + + def sock_connect(self, sock, address): + raise NotImplementedError + + def sock_accept(self, sock): + raise NotImplementedError + + # Signal handling. + + def add_signal_handler(self, sig, callback, *args): + raise NotImplementedError + + def remove_signal_handler(self, sig): + raise NotImplementedError + + # Task factory. + + def set_task_factory(self, factory): + raise NotImplementedError + + def get_task_factory(self): + raise NotImplementedError + + # Error handlers. + + def get_exception_handler(self): + raise NotImplementedError + + def set_exception_handler(self, handler): + raise NotImplementedError + + def default_exception_handler(self, context): + raise NotImplementedError + + def call_exception_handler(self, context): + raise NotImplementedError + + # Debug flag management. + + def get_debug(self): + raise NotImplementedError + + def set_debug(self, enabled): + raise NotImplementedError + + +class AbstractEventLoopPolicy: + """Abstract policy for accessing the event loop.""" + + def get_event_loop(self): + """Get the event loop for the current context. + + Returns an event loop object implementing the BaseEventLoop interface, + or raises an exception in case no event loop has been set for the + current context and the current policy does not specify to create one. + + It should never return None.""" + raise NotImplementedError + + def set_event_loop(self, loop): + """Set the event loop for the current context to loop.""" + raise NotImplementedError + + def new_event_loop(self): + """Create and return a new event loop object according to this + policy's rules. If there's need to set this loop as the event loop for + the current context, set_event_loop must be called explicitly.""" + raise NotImplementedError + + # Child processes handling (Unix only). + + def get_child_watcher(self): + "Get the watcher for child processes." + raise NotImplementedError + + def set_child_watcher(self, watcher): + """Set the watcher for child processes.""" + raise NotImplementedError + + +class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): + """Default policy implementation for accessing the event loop. + + In this policy, each thread has its own event loop. However, we + only automatically create an event loop by default for the main + thread; other threads by default have no event loop. + + Other policies may have different rules (e.g. a single global + event loop, or automatically creating an event loop per thread, or + using some other notion of context to which an event loop is + associated). + """ + + _loop_factory = None + + class _Local(threading.local): + _loop = None + _set_called = False + + def __init__(self): + self._local = self._Local() + + def get_event_loop(self): + """Get the event loop. + + This may be None or an instance of EventLoop. + """ + if (self._local._loop is None and + not self._local._set_called and + isinstance(threading.current_thread(), threading._MainThread)): + self.set_event_loop(self.new_event_loop()) + if self._local._loop is None: + raise RuntimeError('There is no current event loop in thread %r.' + % threading.current_thread().name) + return self._local._loop + + def set_event_loop(self, loop): + """Set the event loop.""" + self._local._set_called = True + assert loop is None or isinstance(loop, AbstractEventLoop) + self._local._loop = loop + + def new_event_loop(self): + """Create a new event loop. + + You must call set_event_loop() to make this the current event + loop. + """ + return self._loop_factory() + + +# Event loop policy. The policy itself is always global, even if the +# policy's rules say that there is an event loop per thread (or other +# notion of context). The default policy is installed by the first +# call to get_event_loop_policy(). +_event_loop_policy = None + +# Lock for protecting the on-the-fly creation of the event loop policy. +_lock = threading.Lock() + + +# A TLS for the running event loop, used by _get_running_loop. +class _RunningLoop(threading.local): + _loop = None +_running_loop = _RunningLoop() + + +def _get_running_loop(): + """Return the running event loop or None. + + This is a low-level function intended to be used by event loops. + This function is thread-specific. + """ + return _running_loop._loop + + +def _set_running_loop(loop): + """Set the running event loop. + + This is a low-level function intended to be used by event loops. + This function is thread-specific. + """ + _running_loop._loop = loop + + +def _init_event_loop_policy(): + global _event_loop_policy + with _lock: + if _event_loop_policy is None: # pragma: no branch + from . import DefaultEventLoopPolicy + _event_loop_policy = DefaultEventLoopPolicy() + + +def get_event_loop_policy(): + """Get the current event loop policy.""" + if _event_loop_policy is None: + _init_event_loop_policy() + return _event_loop_policy + + +def set_event_loop_policy(policy): + """Set the current event loop policy. + + If policy is None, the default policy is restored.""" + global _event_loop_policy + assert policy is None or isinstance(policy, AbstractEventLoopPolicy) + _event_loop_policy = policy + + +def get_event_loop(): + """Return an asyncio event loop. + + When called from a coroutine or a callback (e.g. scheduled with call_soon + or similar API), this function will always return the running event loop. + + If there is no running event loop set, the function will return + the result of `get_event_loop_policy().get_event_loop()` call. + """ + current_loop = _get_running_loop() + if current_loop is not None: + return current_loop + return get_event_loop_policy().get_event_loop() + + +def set_event_loop(loop): + """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" + get_event_loop_policy().set_event_loop(loop) + + +def new_event_loop(): + """Equivalent to calling get_event_loop_policy().new_event_loop().""" + return get_event_loop_policy().new_event_loop() + + +def get_child_watcher(): + """Equivalent to calling get_event_loop_policy().get_child_watcher().""" + return get_event_loop_policy().get_child_watcher() + + +def set_child_watcher(watcher): + """Equivalent to calling + get_event_loop_policy().set_child_watcher(watcher).""" + return get_event_loop_policy().set_child_watcher(watcher) diff --git a/thirdparty/asyncio/asyncio/futures.py b/thirdparty/asyncio/asyncio/futures.py new file mode 100644 index 0000000..bcd4d16 --- /dev/null +++ b/thirdparty/asyncio/asyncio/futures.py @@ -0,0 +1,478 @@ +"""A Future class similar to the one in PEP 3148.""" + +__all__ = ['CancelledError', 'TimeoutError', + 'InvalidStateError', + 'Future', 'wrap_future', + ] + +import concurrent.futures._base +import logging +import reprlib +import sys +import traceback + +from . import compat +from . import events + +# States for Future. +_PENDING = 'PENDING' +_CANCELLED = 'CANCELLED' +_FINISHED = 'FINISHED' + +Error = concurrent.futures._base.Error +CancelledError = concurrent.futures.CancelledError +TimeoutError = concurrent.futures.TimeoutError + +STACK_DEBUG = logging.DEBUG - 1 # heavy-duty debugging + + +class InvalidStateError(Error): + """The operation is not allowed in this state.""" + + +class _TracebackLogger: + """Helper to log a traceback upon destruction if not cleared. + + This solves a nasty problem with Futures and Tasks that have an + exception set: if nobody asks for the exception, the exception is + never logged. This violates the Zen of Python: 'Errors should + never pass silently. Unless explicitly silenced.' + + However, we don't want to log the exception as soon as + set_exception() is called: if the calling code is written + properly, it will get the exception and handle it properly. But + we *do* want to log it if result() or exception() was never called + -- otherwise developers waste a lot of time wondering why their + buggy code fails silently. + + An earlier attempt added a __del__() method to the Future class + itself, but this backfired because the presence of __del__() + prevents garbage collection from breaking cycles. A way out of + this catch-22 is to avoid having a __del__() method on the Future + class itself, but instead to have a reference to a helper object + with a __del__() method that logs the traceback, where we ensure + that the helper object doesn't participate in cycles, and only the + Future has a reference to it. + + The helper object is added when set_exception() is called. When + the Future is collected, and the helper is present, the helper + object is also collected, and its __del__() method will log the + traceback. When the Future's result() or exception() method is + called (and a helper object is present), it removes the helper + object, after calling its clear() method to prevent it from + logging. + + One downside is that we do a fair amount of work to extract the + traceback from the exception, even when it is never logged. It + would seem cheaper to just store the exception object, but that + references the traceback, which references stack frames, which may + reference the Future, which references the _TracebackLogger, and + then the _TracebackLogger would be included in a cycle, which is + what we're trying to avoid! As an optimization, we don't + immediately format the exception; we only do the work when + activate() is called, which call is delayed until after all the + Future's callbacks have run. Since usually a Future has at least + one callback (typically set by 'yield from') and usually that + callback extracts the callback, thereby removing the need to + format the exception. + + PS. I don't claim credit for this solution. I first heard of it + in a discussion about closing files when they are collected. + """ + + __slots__ = ('loop', 'source_traceback', 'exc', 'tb') + + def __init__(self, future, exc): + self.loop = future._loop + self.source_traceback = future._source_traceback + self.exc = exc + self.tb = None + + def activate(self): + exc = self.exc + if exc is not None: + self.exc = None + self.tb = traceback.format_exception(exc.__class__, exc, + exc.__traceback__) + + def clear(self): + self.exc = None + self.tb = None + + def __del__(self): + if self.tb: + msg = 'Future/Task exception was never retrieved\n' + if self.source_traceback: + src = ''.join(traceback.format_list(self.source_traceback)) + msg += 'Future/Task created at (most recent call last):\n' + msg += '%s\n' % src.rstrip() + msg += ''.join(self.tb).rstrip() + self.loop.call_exception_handler({'message': msg}) + + +def isfuture(obj): + """Check for a Future. + + This returns True when obj is a Future instance or is advertising + itself as duck-type compatible by setting _asyncio_future_blocking. + See comment in Future for more details. + """ + return getattr(obj, '_asyncio_future_blocking', None) is not None + + +class Future: + """This class is *almost* compatible with concurrent.futures.Future. + + Differences: + + - result() and exception() do not take a timeout argument and + raise an exception when the future isn't done yet. + + - Callbacks registered with add_done_callback() are always called + via the event loop's call_soon_threadsafe(). + + - This class is not compatible with the wait() and as_completed() + methods in the concurrent.futures package. + + (In Python 3.4 or later we may be able to unify the implementations.) + """ + + # Class variables serving as defaults for instance variables. + _state = _PENDING + _result = None + _exception = None + _loop = None + _source_traceback = None + + # This field is used for a dual purpose: + # - Its presence is a marker to declare that a class implements + # the Future protocol (i.e. is intended to be duck-type compatible). + # The value must also be not-None, to enable a subclass to declare + # that it is not compatible by setting this to None. + # - It is set by __iter__() below so that Task._step() can tell + # the difference between `yield from Future()` (correct) vs. + # `yield Future()` (incorrect). + _asyncio_future_blocking = False + + _log_traceback = False # Used for Python 3.4 and later + _tb_logger = None # Used for Python 3.3 only + + def __init__(self, *, loop=None): + """Initialize the future. + + The optional event_loop argument allows explicitly setting the event + loop object used by the future. If it's not provided, the future uses + the default event loop. + """ + if loop is None: + self._loop = events.get_event_loop() + else: + self._loop = loop + self._callbacks = [] + if self._loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + def __format_callbacks(self): + cb = self._callbacks + size = len(cb) + if not size: + cb = '' + + def format_cb(callback): + return events._format_callback_source(callback, ()) + + if size == 1: + cb = format_cb(cb[0]) + elif size == 2: + cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1])) + elif size > 2: + cb = '{}, <{} more>, {}'.format(format_cb(cb[0]), + size-2, + format_cb(cb[-1])) + return 'cb=[%s]' % cb + + def _repr_info(self): + info = [self._state.lower()] + if self._state == _FINISHED: + if self._exception is not None: + info.append('exception={!r}'.format(self._exception)) + else: + # use reprlib to limit the length of the output, especially + # for very long strings + result = reprlib.repr(self._result) + info.append('result={}'.format(result)) + if self._callbacks: + info.append(self.__format_callbacks()) + if self._source_traceback: + frame = self._source_traceback[-1] + info.append('created at %s:%s' % (frame[0], frame[1])) + return info + + def __repr__(self): + info = self._repr_info() + return '<%s %s>' % (self.__class__.__name__, ' '.join(info)) + + # On Python 3.3 and older, objects with a destructor part of a reference + # cycle are never destroyed. It's not more the case on Python 3.4 thanks + # to the PEP 442. + if compat.PY34: + def __del__(self): + if not self._log_traceback: + # set_exception() was not called, or result() or exception() + # has consumed the exception + return + exc = self._exception + context = { + 'message': ('%s exception was never retrieved' + % self.__class__.__name__), + 'exception': exc, + 'future': self, + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + + def cancel(self): + """Cancel the future and schedule callbacks. + + If the future is already done or cancelled, return False. Otherwise, + change the future's state to cancelled, schedule the callbacks and + return True. + """ + if self._state != _PENDING: + return False + self._state = _CANCELLED + self._schedule_callbacks() + return True + + def _schedule_callbacks(self): + """Internal: Ask the event loop to call all callbacks. + + The callbacks are scheduled to be called as soon as possible. Also + clears the callback list. + """ + callbacks = self._callbacks[:] + if not callbacks: + return + + self._callbacks[:] = [] + for callback in callbacks: + self._loop.call_soon(callback, self) + + def cancelled(self): + """Return True if the future was cancelled.""" + return self._state == _CANCELLED + + # Don't implement running(); see http://bugs.python.org/issue18699 + + def done(self): + """Return True if the future is done. + + Done means either that a result / exception are available, or that the + future was cancelled. + """ + return self._state != _PENDING + + def result(self): + """Return the result this future represents. + + If the future has been cancelled, raises CancelledError. If the + future's result isn't yet available, raises InvalidStateError. If + the future is done and has an exception set, this exception is raised. + """ + if self._state == _CANCELLED: + raise CancelledError + if self._state != _FINISHED: + raise InvalidStateError('Result is not ready.') + self._log_traceback = False + if self._tb_logger is not None: + self._tb_logger.clear() + self._tb_logger = None + if self._exception is not None: + raise self._exception + return self._result + + def exception(self): + """Return the exception that was set on this future. + + The exception (or None if no exception was set) is returned only if + the future is done. If the future has been cancelled, raises + CancelledError. If the future isn't done yet, raises + InvalidStateError. + """ + if self._state == _CANCELLED: + raise CancelledError + if self._state != _FINISHED: + raise InvalidStateError('Exception is not set.') + self._log_traceback = False + if self._tb_logger is not None: + self._tb_logger.clear() + self._tb_logger = None + return self._exception + + def add_done_callback(self, fn): + """Add a callback to be run when the future becomes done. + + The callback is called with a single argument - the future object. If + the future is already done when this is called, the callback is + scheduled with call_soon. + """ + if self._state != _PENDING: + self._loop.call_soon(fn, self) + else: + self._callbacks.append(fn) + + # New method not in PEP 3148. + + def remove_done_callback(self, fn): + """Remove all instances of a callback from the "call when done" list. + + Returns the number of callbacks removed. + """ + filtered_callbacks = [f for f in self._callbacks if f != fn] + removed_count = len(self._callbacks) - len(filtered_callbacks) + if removed_count: + self._callbacks[:] = filtered_callbacks + return removed_count + + # So-called internal methods (note: no set_running_or_notify_cancel()). + + def set_result(self, result): + """Mark the future done and set its result. + + If the future is already done when this method is called, raises + InvalidStateError. + """ + if self._state != _PENDING: + raise InvalidStateError('{}: {!r}'.format(self._state, self)) + self._result = result + self._state = _FINISHED + self._schedule_callbacks() + + def set_exception(self, exception): + """Mark the future done and set an exception. + + If the future is already done when this method is called, raises + InvalidStateError. + """ + if self._state != _PENDING: + raise InvalidStateError('{}: {!r}'.format(self._state, self)) + if isinstance(exception, type): + exception = exception() + if type(exception) is StopIteration: + raise TypeError("StopIteration interacts badly with generators " + "and cannot be raised into a Future") + self._exception = exception + self._state = _FINISHED + self._schedule_callbacks() + if compat.PY34: + self._log_traceback = True + else: + self._tb_logger = _TracebackLogger(self, exception) + # Arrange for the logger to be activated after all callbacks + # have had a chance to call result() or exception(). + self._loop.call_soon(self._tb_logger.activate) + + def __iter__(self): + if not self.done(): + self._asyncio_future_blocking = True + yield self # This tells Task to wait for completion. + assert self.done(), "yield from wasn't used with future" + return self.result() # May raise too. + + if compat.PY35: + __await__ = __iter__ # make compatible with 'await' expression + + +def _set_result_unless_cancelled(fut, result): + """Helper setting the result only if the future was not cancelled.""" + if fut.cancelled(): + return + fut.set_result(result) + + +def _set_concurrent_future_state(concurrent, source): + """Copy state from a future to a concurrent.futures.Future.""" + assert source.done() + if source.cancelled(): + concurrent.cancel() + if not concurrent.set_running_or_notify_cancel(): + return + exception = source.exception() + if exception is not None: + concurrent.set_exception(exception) + else: + result = source.result() + concurrent.set_result(result) + + +def _copy_future_state(source, dest): + """Internal helper to copy state from another Future. + + The other Future may be a concurrent.futures.Future. + """ + assert source.done() + if dest.cancelled(): + return + assert not dest.done() + if source.cancelled(): + dest.cancel() + else: + exception = source.exception() + if exception is not None: + dest.set_exception(exception) + else: + result = source.result() + dest.set_result(result) + + +def _chain_future(source, destination): + """Chain two futures so that when one completes, so does the other. + + The result (or exception) of source will be copied to destination. + If destination is cancelled, source gets cancelled too. + Compatible with both asyncio.Future and concurrent.futures.Future. + """ + if not isfuture(source) and not isinstance(source, + concurrent.futures.Future): + raise TypeError('A future is required for source argument') + if not isfuture(destination) and not isinstance(destination, + concurrent.futures.Future): + raise TypeError('A future is required for destination argument') + source_loop = source._loop if isfuture(source) else None + dest_loop = destination._loop if isfuture(destination) else None + + def _set_state(future, other): + if isfuture(future): + _copy_future_state(other, future) + else: + _set_concurrent_future_state(future, other) + + def _call_check_cancel(destination): + if destination.cancelled(): + if source_loop is None or source_loop is dest_loop: + source.cancel() + else: + source_loop.call_soon_threadsafe(source.cancel) + + def _call_set_state(source): + if dest_loop is None or dest_loop is source_loop: + _set_state(destination, source) + else: + dest_loop.call_soon_threadsafe(_set_state, destination, source) + + destination.add_done_callback(_call_check_cancel) + source.add_done_callback(_call_set_state) + + +def wrap_future(future, *, loop=None): + """Wrap concurrent.futures.Future object.""" + if isfuture(future): + return future + assert isinstance(future, concurrent.futures.Future), \ + 'concurrent.futures.Future is expected, got {!r}'.format(future) + if loop is None: + loop = events.get_event_loop() + new_future = loop.create_future() + _chain_future(future, new_future) + return new_future diff --git a/thirdparty/asyncio/asyncio/locks.py b/thirdparty/asyncio/asyncio/locks.py new file mode 100644 index 0000000..deefc93 --- /dev/null +++ b/thirdparty/asyncio/asyncio/locks.py @@ -0,0 +1,478 @@ +"""Synchronization primitives.""" + +__all__ = ['Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore'] + +import collections + +from . import compat +from . import events +from . import futures +from .coroutines import coroutine + + +class _ContextManager: + """Context manager. + + This enables the following idiom for acquiring and releasing a + lock around a block: + + with (yield from lock): + + + while failing loudly when accidentally using: + + with lock: + + """ + + def __init__(self, lock): + self._lock = lock + + def __enter__(self): + # We have no use for the "as ..." clause in the with + # statement for locks. + return None + + def __exit__(self, *args): + try: + self._lock.release() + finally: + self._lock = None # Crudely prevent reuse. + + +class _ContextManagerMixin: + def __enter__(self): + raise RuntimeError( + '"yield from" should be used as context manager expression') + + def __exit__(self, *args): + # This must exist because __enter__ exists, even though that + # always raises; that's how the with-statement works. + pass + + @coroutine + def __iter__(self): + # This is not a coroutine. It is meant to enable the idiom: + # + # with (yield from lock): + # + # + # as an alternative to: + # + # yield from lock.acquire() + # try: + # + # finally: + # lock.release() + yield from self.acquire() + return _ContextManager(self) + + if compat.PY35: + + def __await__(self): + # To make "with await lock" work. + yield from self.acquire() + return _ContextManager(self) + + @coroutine + def __aenter__(self): + yield from self.acquire() + # We have no use for the "as ..." clause in the with + # statement for locks. + return None + + @coroutine + def __aexit__(self, exc_type, exc, tb): + self.release() + + +class Lock(_ContextManagerMixin): + """Primitive lock objects. + + A primitive lock is a synchronization primitive that is not owned + by a particular coroutine when locked. A primitive lock is in one + of two states, 'locked' or 'unlocked'. + + It is created in the unlocked state. It has two basic methods, + acquire() and release(). When the state is unlocked, acquire() + changes the state to locked and returns immediately. When the + state is locked, acquire() blocks until a call to release() in + another coroutine changes it to unlocked, then the acquire() call + resets it to locked and returns. The release() method should only + be called in the locked state; it changes the state to unlocked + and returns immediately. If an attempt is made to release an + unlocked lock, a RuntimeError will be raised. + + When more than one coroutine is blocked in acquire() waiting for + the state to turn to unlocked, only one coroutine proceeds when a + release() call resets the state to unlocked; first coroutine which + is blocked in acquire() is being processed. + + acquire() is a coroutine and should be called with 'yield from'. + + Locks also support the context management protocol. '(yield from lock)' + should be used as the context manager expression. + + Usage: + + lock = Lock() + ... + yield from lock + try: + ... + finally: + lock.release() + + Context manager usage: + + lock = Lock() + ... + with (yield from lock): + ... + + Lock objects can be tested for locking state: + + if not lock.locked(): + yield from lock + else: + # lock is acquired + ... + + """ + + def __init__(self, *, loop=None): + self._waiters = collections.deque() + self._locked = False + if loop is not None: + self._loop = loop + else: + self._loop = events.get_event_loop() + + def __repr__(self): + res = super().__repr__() + extra = 'locked' if self._locked else 'unlocked' + if self._waiters: + extra = '{},waiters:{}'.format(extra, len(self._waiters)) + return '<{} [{}]>'.format(res[1:-1], extra) + + def locked(self): + """Return True if lock is acquired.""" + return self._locked + + @coroutine + def acquire(self): + """Acquire a lock. + + This method blocks until the lock is unlocked, then sets it to + locked and returns True. + """ + if not self._locked and all(w.cancelled() for w in self._waiters): + self._locked = True + return True + + fut = self._loop.create_future() + self._waiters.append(fut) + try: + yield from fut + self._locked = True + return True + finally: + self._waiters.remove(fut) + + def release(self): + """Release a lock. + + When the lock is locked, reset it to unlocked, and return. + If any other coroutines are blocked waiting for the lock to become + unlocked, allow exactly one of them to proceed. + + When invoked on an unlocked lock, a RuntimeError is raised. + + There is no return value. + """ + if self._locked: + self._locked = False + # Wake up the first waiter who isn't cancelled. + for fut in self._waiters: + if not fut.done(): + fut.set_result(True) + break + else: + raise RuntimeError('Lock is not acquired.') + + +class Event: + """Asynchronous equivalent to threading.Event. + + Class implementing event objects. An event manages a flag that can be set + to true with the set() method and reset to false with the clear() method. + The wait() method blocks until the flag is true. The flag is initially + false. + """ + + def __init__(self, *, loop=None): + self._waiters = collections.deque() + self._value = False + if loop is not None: + self._loop = loop + else: + self._loop = events.get_event_loop() + + def __repr__(self): + res = super().__repr__() + extra = 'set' if self._value else 'unset' + if self._waiters: + extra = '{},waiters:{}'.format(extra, len(self._waiters)) + return '<{} [{}]>'.format(res[1:-1], extra) + + def is_set(self): + """Return True if and only if the internal flag is true.""" + return self._value + + def set(self): + """Set the internal flag to true. All coroutines waiting for it to + become true are awakened. Coroutine that call wait() once the flag is + true will not block at all. + """ + if not self._value: + self._value = True + + for fut in self._waiters: + if not fut.done(): + fut.set_result(True) + + def clear(self): + """Reset the internal flag to false. Subsequently, coroutines calling + wait() will block until set() is called to set the internal flag + to true again.""" + self._value = False + + @coroutine + def wait(self): + """Block until the internal flag is true. + + If the internal flag is true on entry, return True + immediately. Otherwise, block until another coroutine calls + set() to set the flag to true, then return True. + """ + if self._value: + return True + + fut = self._loop.create_future() + self._waiters.append(fut) + try: + yield from fut + return True + finally: + self._waiters.remove(fut) + + +class Condition(_ContextManagerMixin): + """Asynchronous equivalent to threading.Condition. + + This class implements condition variable objects. A condition variable + allows one or more coroutines to wait until they are notified by another + coroutine. + + A new Lock object is created and used as the underlying lock. + """ + + def __init__(self, lock=None, *, loop=None): + if loop is not None: + self._loop = loop + else: + self._loop = events.get_event_loop() + + if lock is None: + lock = Lock(loop=self._loop) + elif lock._loop is not self._loop: + raise ValueError("loop argument must agree with lock") + + self._lock = lock + # Export the lock's locked(), acquire() and release() methods. + self.locked = lock.locked + self.acquire = lock.acquire + self.release = lock.release + + self._waiters = collections.deque() + + def __repr__(self): + res = super().__repr__() + extra = 'locked' if self.locked() else 'unlocked' + if self._waiters: + extra = '{},waiters:{}'.format(extra, len(self._waiters)) + return '<{} [{}]>'.format(res[1:-1], extra) + + @coroutine + def wait(self): + """Wait until notified. + + If the calling coroutine has not acquired the lock when this + method is called, a RuntimeError is raised. + + This method releases the underlying lock, and then blocks + until it is awakened by a notify() or notify_all() call for + the same condition variable in another coroutine. Once + awakened, it re-acquires the lock and returns True. + """ + if not self.locked(): + raise RuntimeError('cannot wait on un-acquired lock') + + self.release() + try: + fut = self._loop.create_future() + self._waiters.append(fut) + try: + yield from fut + return True + finally: + self._waiters.remove(fut) + + finally: + # Must reacquire lock even if wait is cancelled + while True: + try: + yield from self.acquire() + break + except futures.CancelledError: + pass + + @coroutine + def wait_for(self, predicate): + """Wait until a predicate becomes true. + + The predicate should be a callable which result will be + interpreted as a boolean value. The final predicate value is + the return value. + """ + result = predicate() + while not result: + yield from self.wait() + result = predicate() + return result + + def notify(self, n=1): + """By default, wake up one coroutine waiting on this condition, if any. + If the calling coroutine has not acquired the lock when this method + is called, a RuntimeError is raised. + + This method wakes up at most n of the coroutines waiting for the + condition variable; it is a no-op if no coroutines are waiting. + + Note: an awakened coroutine does not actually return from its + wait() call until it can reacquire the lock. Since notify() does + not release the lock, its caller should. + """ + if not self.locked(): + raise RuntimeError('cannot notify on un-acquired lock') + + idx = 0 + for fut in self._waiters: + if idx >= n: + break + + if not fut.done(): + idx += 1 + fut.set_result(False) + + def notify_all(self): + """Wake up all threads waiting on this condition. This method acts + like notify(), but wakes up all waiting threads instead of one. If the + calling thread has not acquired the lock when this method is called, + a RuntimeError is raised. + """ + self.notify(len(self._waiters)) + + +class Semaphore(_ContextManagerMixin): + """A Semaphore implementation. + + A semaphore manages an internal counter which is decremented by each + acquire() call and incremented by each release() call. The counter + can never go below zero; when acquire() finds that it is zero, it blocks, + waiting until some other thread calls release(). + + Semaphores also support the context management protocol. + + The optional argument gives the initial value for the internal + counter; it defaults to 1. If the value given is less than 0, + ValueError is raised. + """ + + def __init__(self, value=1, *, loop=None): + if value < 0: + raise ValueError("Semaphore initial value must be >= 0") + self._value = value + self._waiters = collections.deque() + if loop is not None: + self._loop = loop + else: + self._loop = events.get_event_loop() + + def __repr__(self): + res = super().__repr__() + extra = 'locked' if self.locked() else 'unlocked,value:{}'.format( + self._value) + if self._waiters: + extra = '{},waiters:{}'.format(extra, len(self._waiters)) + return '<{} [{}]>'.format(res[1:-1], extra) + + def _wake_up_next(self): + while self._waiters: + waiter = self._waiters.popleft() + if not waiter.done(): + waiter.set_result(None) + return + + def locked(self): + """Returns True if semaphore can not be acquired immediately.""" + return self._value == 0 + + @coroutine + def acquire(self): + """Acquire a semaphore. + + If the internal counter is larger than zero on entry, + decrement it by one and return True immediately. If it is + zero on entry, block, waiting until some other coroutine has + called release() to make it larger than 0, and then return + True. + """ + while self._value <= 0: + fut = self._loop.create_future() + self._waiters.append(fut) + try: + yield from fut + except: + # See the similar code in Queue.get. + fut.cancel() + if self._value > 0 and not fut.cancelled(): + self._wake_up_next() + raise + self._value -= 1 + return True + + def release(self): + """Release a semaphore, incrementing the internal counter by one. + When it was zero on entry and another coroutine is waiting for it to + become larger than zero again, wake up that coroutine. + """ + self._value += 1 + self._wake_up_next() + + +class BoundedSemaphore(Semaphore): + """A bounded semaphore implementation. + + This raises ValueError in release() if it would increase the value + above the initial value. + """ + + def __init__(self, value=1, *, loop=None): + self._bound_value = value + super().__init__(value, loop=loop) + + def release(self): + if self._value >= self._bound_value: + raise ValueError('BoundedSemaphore released too many times') + super().release() diff --git a/thirdparty/asyncio/asyncio/log.py b/thirdparty/asyncio/asyncio/log.py new file mode 100644 index 0000000..23a7074 --- /dev/null +++ b/thirdparty/asyncio/asyncio/log.py @@ -0,0 +1,7 @@ +"""Logging configuration.""" + +import logging + + +# Name the logger after the package. +logger = logging.getLogger(__package__) diff --git a/thirdparty/asyncio/asyncio/proactor_events.py b/thirdparty/asyncio/asyncio/proactor_events.py new file mode 100644 index 0000000..fef3205 --- /dev/null +++ b/thirdparty/asyncio/asyncio/proactor_events.py @@ -0,0 +1,549 @@ +"""Event loop using a proactor and related classes. + +A proactor is a "notify-on-completion" multiplexer. Currently a +proactor is only implemented on Windows with IOCP. +""" + +__all__ = ['BaseProactorEventLoop'] + +import socket +import warnings + +from . import base_events +from . import compat +from . import constants +from . import futures +from . import sslproto +from . import transports +from .log import logger + + +class _ProactorBasePipeTransport(transports._FlowControlMixin, + transports.BaseTransport): + """Base class for pipe and socket transports.""" + + def __init__(self, loop, sock, protocol, waiter=None, + extra=None, server=None): + super().__init__(extra, loop) + self._set_extra(sock) + self._sock = sock + self._protocol = protocol + self._server = server + self._buffer = None # None or bytearray. + self._read_fut = None + self._write_fut = None + self._pending_write = 0 + self._conn_lost = 0 + self._closing = False # Set when close() called. + self._eof_written = False + if self._server is not None: + self._server._attach() + self._loop.call_soon(self._protocol.connection_made, self) + if waiter is not None: + # only wake up the waiter when connection_made() has been called + self._loop.call_soon(futures._set_result_unless_cancelled, + waiter, None) + + def __repr__(self): + info = [self.__class__.__name__] + if self._sock is None: + info.append('closed') + elif self._closing: + info.append('closing') + if self._sock is not None: + info.append('fd=%s' % self._sock.fileno()) + if self._read_fut is not None: + info.append('read=%s' % self._read_fut) + if self._write_fut is not None: + info.append("write=%r" % self._write_fut) + if self._buffer: + bufsize = len(self._buffer) + info.append('write_bufsize=%s' % bufsize) + if self._eof_written: + info.append('EOF written') + return '<%s>' % ' '.join(info) + + def _set_extra(self, sock): + self._extra['pipe'] = sock + + def set_protocol(self, protocol): + self._protocol = protocol + + def get_protocol(self): + return self._protocol + + def is_closing(self): + return self._closing + + def close(self): + if self._closing: + return + self._closing = True + self._conn_lost += 1 + if not self._buffer and self._write_fut is None: + self._loop.call_soon(self._call_connection_lost, None) + if self._read_fut is not None: + self._read_fut.cancel() + self._read_fut = None + + # On Python 3.3 and older, objects with a destructor part of a reference + # cycle are never destroyed. It's not more the case on Python 3.4 thanks + # to the PEP 442. + if compat.PY34: + def __del__(self): + if self._sock is not None: + warnings.warn("unclosed transport %r" % self, ResourceWarning) + self.close() + + def _fatal_error(self, exc, message='Fatal error on pipe transport'): + if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if self._loop.get_debug(): + logger.debug("%r: %s", self, message, exc_info=True) + else: + self._loop.call_exception_handler({ + 'message': message, + 'exception': exc, + 'transport': self, + 'protocol': self._protocol, + }) + self._force_close(exc) + + def _force_close(self, exc): + if self._closing: + return + self._closing = True + self._conn_lost += 1 + if self._write_fut: + self._write_fut.cancel() + self._write_fut = None + if self._read_fut: + self._read_fut.cancel() + self._read_fut = None + self._pending_write = 0 + self._buffer = None + self._loop.call_soon(self._call_connection_lost, exc) + + def _call_connection_lost(self, exc): + try: + self._protocol.connection_lost(exc) + finally: + # XXX If there is a pending overlapped read on the other + # end then it may fail with ERROR_NETNAME_DELETED if we + # just close our end. First calling shutdown() seems to + # cure it, but maybe using DisconnectEx() would be better. + if hasattr(self._sock, 'shutdown'): + self._sock.shutdown(socket.SHUT_RDWR) + self._sock.close() + self._sock = None + server = self._server + if server is not None: + server._detach() + self._server = None + + def get_write_buffer_size(self): + size = self._pending_write + if self._buffer is not None: + size += len(self._buffer) + return size + + +class _ProactorReadPipeTransport(_ProactorBasePipeTransport, + transports.ReadTransport): + """Transport for read pipes.""" + + def __init__(self, loop, sock, protocol, waiter=None, + extra=None, server=None): + super().__init__(loop, sock, protocol, waiter, extra, server) + self._paused = False + self._loop.call_soon(self._loop_reading) + + def pause_reading(self): + if self._closing: + raise RuntimeError('Cannot pause_reading() when closing') + if self._paused: + raise RuntimeError('Already paused') + self._paused = True + if self._loop.get_debug(): + logger.debug("%r pauses reading", self) + + def resume_reading(self): + if not self._paused: + raise RuntimeError('Not paused') + self._paused = False + if self._closing: + return + self._loop.call_soon(self._loop_reading, self._read_fut) + if self._loop.get_debug(): + logger.debug("%r resumes reading", self) + + def _loop_reading(self, fut=None): + if self._paused: + return + data = None + + try: + if fut is not None: + assert self._read_fut is fut or (self._read_fut is None and + self._closing) + self._read_fut = None + data = fut.result() # deliver data later in "finally" clause + + if self._closing: + # since close() has been called we ignore any read data + data = None + return + + if data == b'': + # we got end-of-file so no need to reschedule a new read + return + + # reschedule a new read + self._read_fut = self._loop._proactor.recv(self._sock, 4096) + except ConnectionAbortedError as exc: + if not self._closing: + self._fatal_error(exc, 'Fatal read error on pipe transport') + elif self._loop.get_debug(): + logger.debug("Read error on pipe transport while closing", + exc_info=True) + except ConnectionResetError as exc: + self._force_close(exc) + except OSError as exc: + self._fatal_error(exc, 'Fatal read error on pipe transport') + except futures.CancelledError: + if not self._closing: + raise + else: + self._read_fut.add_done_callback(self._loop_reading) + finally: + if data: + self._protocol.data_received(data) + elif data is not None: + if self._loop.get_debug(): + logger.debug("%r received EOF", self) + keep_open = self._protocol.eof_received() + if not keep_open: + self.close() + + +class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, + transports.WriteTransport): + """Transport for write pipes.""" + + def write(self, data): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be byte-ish (%r)', + type(data)) + if self._eof_written: + raise RuntimeError('write_eof() already called') + + if not data: + return + + if self._conn_lost: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.send() raised exception.') + self._conn_lost += 1 + return + + # Observable states: + # 1. IDLE: _write_fut and _buffer both None + # 2. WRITING: _write_fut set; _buffer None + # 3. BACKED UP: _write_fut set; _buffer a bytearray + # We always copy the data, so the caller can't modify it + # while we're still waiting for the I/O to happen. + if self._write_fut is None: # IDLE -> WRITING + assert self._buffer is None + # Pass a copy, except if it's already immutable. + self._loop_writing(data=bytes(data)) + elif not self._buffer: # WRITING -> BACKED UP + # Make a mutable copy which we can extend. + self._buffer = bytearray(data) + self._maybe_pause_protocol() + else: # BACKED UP + # Append to buffer (also copies). + self._buffer.extend(data) + self._maybe_pause_protocol() + + def _loop_writing(self, f=None, data=None): + try: + assert f is self._write_fut + self._write_fut = None + self._pending_write = 0 + if f: + f.result() + if data is None: + data = self._buffer + self._buffer = None + if not data: + if self._closing: + self._loop.call_soon(self._call_connection_lost, None) + if self._eof_written: + self._sock.shutdown(socket.SHUT_WR) + # Now that we've reduced the buffer size, tell the + # protocol to resume writing if it was paused. Note that + # we do this last since the callback is called immediately + # and it may add more data to the buffer (even causing the + # protocol to be paused again). + self._maybe_resume_protocol() + else: + self._write_fut = self._loop._proactor.send(self._sock, data) + if not self._write_fut.done(): + assert self._pending_write == 0 + self._pending_write = len(data) + self._write_fut.add_done_callback(self._loop_writing) + self._maybe_pause_protocol() + else: + self._write_fut.add_done_callback(self._loop_writing) + except ConnectionResetError as exc: + self._force_close(exc) + except OSError as exc: + self._fatal_error(exc, 'Fatal write error on pipe transport') + + def can_write_eof(self): + return True + + def write_eof(self): + self.close() + + def abort(self): + self._force_close(None) + + +class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport): + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self._read_fut = self._loop._proactor.recv(self._sock, 16) + self._read_fut.add_done_callback(self._pipe_closed) + + def _pipe_closed(self, fut): + if fut.cancelled(): + # the transport has been closed + return + assert fut.result() == b'' + if self._closing: + assert self._read_fut is None + return + assert fut is self._read_fut, (fut, self._read_fut) + self._read_fut = None + if self._write_fut is not None: + self._force_close(BrokenPipeError()) + else: + self.close() + + +class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport, + _ProactorBaseWritePipeTransport, + transports.Transport): + """Transport for duplex pipes.""" + + def can_write_eof(self): + return False + + def write_eof(self): + raise NotImplementedError + + +class _ProactorSocketTransport(_ProactorReadPipeTransport, + _ProactorBaseWritePipeTransport, + transports.Transport): + """Transport for connected sockets.""" + + def _set_extra(self, sock): + self._extra['socket'] = sock + try: + self._extra['sockname'] = sock.getsockname() + except (socket.error, AttributeError): + if self._loop.get_debug(): + logger.warning("getsockname() failed on %r", + sock, exc_info=True) + if 'peername' not in self._extra: + try: + self._extra['peername'] = sock.getpeername() + except (socket.error, AttributeError): + if self._loop.get_debug(): + logger.warning("getpeername() failed on %r", + sock, exc_info=True) + + def can_write_eof(self): + return True + + def write_eof(self): + if self._closing or self._eof_written: + return + self._eof_written = True + if self._write_fut is None: + self._sock.shutdown(socket.SHUT_WR) + + +class BaseProactorEventLoop(base_events.BaseEventLoop): + + def __init__(self, proactor): + super().__init__() + logger.debug('Using proactor: %s', proactor.__class__.__name__) + self._proactor = proactor + self._selector = proactor # convenient alias + self._self_reading_future = None + self._accept_futures = {} # socket file descriptor => Future + proactor.set_loop(self) + self._make_self_pipe() + + def _make_socket_transport(self, sock, protocol, waiter=None, + extra=None, server=None): + return _ProactorSocketTransport(self, sock, protocol, waiter, + extra, server) + + def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None): + if not sslproto._is_sslproto_available(): + raise NotImplementedError("Proactor event loop requires Python 3.5" + " or newer (ssl.MemoryBIO) to support " + "SSL") + + ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, + server_side, server_hostname) + _ProactorSocketTransport(self, rawsock, ssl_protocol, + extra=extra, server=server) + return ssl_protocol._app_transport + + def _make_duplex_pipe_transport(self, sock, protocol, waiter=None, + extra=None): + return _ProactorDuplexPipeTransport(self, + sock, protocol, waiter, extra) + + def _make_read_pipe_transport(self, sock, protocol, waiter=None, + extra=None): + return _ProactorReadPipeTransport(self, sock, protocol, waiter, extra) + + def _make_write_pipe_transport(self, sock, protocol, waiter=None, + extra=None): + # We want connection_lost() to be called when other end closes + return _ProactorWritePipeTransport(self, + sock, protocol, waiter, extra) + + def close(self): + if self.is_running(): + raise RuntimeError("Cannot close a running event loop") + if self.is_closed(): + return + + # Call these methods before closing the event loop (before calling + # BaseEventLoop.close), because they can schedule callbacks with + # call_soon(), which is forbidden when the event loop is closed. + self._stop_accept_futures() + self._close_self_pipe() + self._proactor.close() + self._proactor = None + self._selector = None + + # Close the event loop + super().close() + + def sock_recv(self, sock, n): + return self._proactor.recv(sock, n) + + def sock_sendall(self, sock, data): + return self._proactor.send(sock, data) + + def sock_connect(self, sock, address): + return self._proactor.connect(sock, address) + + def sock_accept(self, sock): + return self._proactor.accept(sock) + + def _socketpair(self): + raise NotImplementedError + + def _close_self_pipe(self): + if self._self_reading_future is not None: + self._self_reading_future.cancel() + self._self_reading_future = None + self._ssock.close() + self._ssock = None + self._csock.close() + self._csock = None + self._internal_fds -= 1 + + def _make_self_pipe(self): + # A self-socket, really. :-) + self._ssock, self._csock = self._socketpair() + self._ssock.setblocking(False) + self._csock.setblocking(False) + self._internal_fds += 1 + self.call_soon(self._loop_self_reading) + + def _loop_self_reading(self, f=None): + try: + if f is not None: + f.result() # may raise + f = self._proactor.recv(self._ssock, 4096) + except futures.CancelledError: + # _close_self_pipe() has been called, stop waiting for data + return + except Exception as exc: + self.call_exception_handler({ + 'message': 'Error on reading from the event loop self pipe', + 'exception': exc, + 'loop': self, + }) + else: + self._self_reading_future = f + f.add_done_callback(self._loop_self_reading) + + def _write_to_self(self): + self._csock.send(b'\0') + + def _start_serving(self, protocol_factory, sock, + sslcontext=None, server=None, backlog=100): + + def loop(f=None): + try: + if f is not None: + conn, addr = f.result() + if self._debug: + logger.debug("%r got a new connection from %r: %r", + server, addr, conn) + protocol = protocol_factory() + if sslcontext is not None: + self._make_ssl_transport( + conn, protocol, sslcontext, server_side=True, + extra={'peername': addr}, server=server) + else: + self._make_socket_transport( + conn, protocol, + extra={'peername': addr}, server=server) + if self.is_closed(): + return + f = self._proactor.accept(sock) + except OSError as exc: + if sock.fileno() != -1: + self.call_exception_handler({ + 'message': 'Accept failed on a socket', + 'exception': exc, + 'socket': sock, + }) + sock.close() + elif self._debug: + logger.debug("Accept failed on socket %r", + sock, exc_info=True) + except futures.CancelledError: + sock.close() + else: + self._accept_futures[sock.fileno()] = f + f.add_done_callback(loop) + + self.call_soon(loop) + + def _process_events(self, event_list): + # Events are processed in the IocpProactor._poll() method + pass + + def _stop_accept_futures(self): + for future in self._accept_futures.values(): + future.cancel() + self._accept_futures.clear() + + def _stop_serving(self, sock): + self._stop_accept_futures() + self._proactor._stop_serving(sock) + sock.close() diff --git a/thirdparty/asyncio/asyncio/protocols.py b/thirdparty/asyncio/asyncio/protocols.py new file mode 100644 index 0000000..80fcac9 --- /dev/null +++ b/thirdparty/asyncio/asyncio/protocols.py @@ -0,0 +1,134 @@ +"""Abstract Protocol class.""" + +__all__ = ['BaseProtocol', 'Protocol', 'DatagramProtocol', + 'SubprocessProtocol'] + + +class BaseProtocol: + """Common base class for protocol interfaces. + + Usually user implements protocols that derived from BaseProtocol + like Protocol or ProcessProtocol. + + The only case when BaseProtocol should be implemented directly is + write-only transport like write pipe + """ + + def connection_made(self, transport): + """Called when a connection is made. + + The argument is the transport representing the pipe connection. + To receive data, wait for data_received() calls. + When the connection is closed, connection_lost() is called. + """ + + def connection_lost(self, exc): + """Called when the connection is lost or closed. + + The argument is an exception object or None (the latter + meaning a regular EOF is received or the connection was + aborted or closed). + """ + + def pause_writing(self): + """Called when the transport's buffer goes over the high-water mark. + + Pause and resume calls are paired -- pause_writing() is called + once when the buffer goes strictly over the high-water mark + (even if subsequent writes increases the buffer size even + more), and eventually resume_writing() is called once when the + buffer size reaches the low-water mark. + + Note that if the buffer size equals the high-water mark, + pause_writing() is not called -- it must go strictly over. + Conversely, resume_writing() is called when the buffer size is + equal or lower than the low-water mark. These end conditions + are important to ensure that things go as expected when either + mark is zero. + + NOTE: This is the only Protocol callback that is not called + through EventLoop.call_soon() -- if it were, it would have no + effect when it's most needed (when the app keeps writing + without yielding until pause_writing() is called). + """ + + def resume_writing(self): + """Called when the transport's buffer drains below the low-water mark. + + See pause_writing() for details. + """ + + +class Protocol(BaseProtocol): + """Interface for stream protocol. + + The user should implement this interface. They can inherit from + this class but don't need to. The implementations here do + nothing (they don't raise exceptions). + + When the user wants to requests a transport, they pass a protocol + factory to a utility function (e.g., EventLoop.create_connection()). + + When the connection is made successfully, connection_made() is + called with a suitable transport object. Then data_received() + will be called 0 or more times with data (bytes) received from the + transport; finally, connection_lost() will be called exactly once + with either an exception object or None as an argument. + + State machine of calls: + + start -> CM [-> DR*] [-> ER?] -> CL -> end + + * CM: connection_made() + * DR: data_received() + * ER: eof_received() + * CL: connection_lost() + """ + + def data_received(self, data): + """Called when some data is received. + + The argument is a bytes object. + """ + + def eof_received(self): + """Called when the other end calls write_eof() or equivalent. + + If this returns a false value (including None), the transport + will close itself. If it returns a true value, closing the + transport is up to the protocol. + """ + + +class DatagramProtocol(BaseProtocol): + """Interface for datagram protocol.""" + + def datagram_received(self, data, addr): + """Called when some datagram is received.""" + + def error_received(self, exc): + """Called when a send or receive operation raises an OSError. + + (Other than BlockingIOError or InterruptedError.) + """ + + +class SubprocessProtocol(BaseProtocol): + """Interface for protocol for subprocess calls.""" + + def pipe_data_received(self, fd, data): + """Called when the subprocess writes data into stdout/stderr pipe. + + fd is int file descriptor. + data is bytes object. + """ + + def pipe_connection_lost(self, fd, exc): + """Called when a file descriptor associated with the child process is + closed. + + fd is the int file descriptor that was closed. + """ + + def process_exited(self): + """Called when subprocess has exited.""" diff --git a/thirdparty/asyncio/asyncio/queues.py b/thirdparty/asyncio/asyncio/queues.py new file mode 100644 index 0000000..2d38972 --- /dev/null +++ b/thirdparty/asyncio/asyncio/queues.py @@ -0,0 +1,253 @@ +"""Queues""" + +__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty'] + +import collections +import heapq + +from . import compat +from . import events +from . import locks +from .coroutines import coroutine + + +class QueueEmpty(Exception): + """Exception raised when Queue.get_nowait() is called on a Queue object + which is empty. + """ + pass + + +class QueueFull(Exception): + """Exception raised when the Queue.put_nowait() method is called on a Queue + object which is full. + """ + pass + + +class Queue: + """A queue, useful for coordinating producer and consumer coroutines. + + If maxsize is less than or equal to zero, the queue size is infinite. If it + is an integer greater than 0, then "yield from put()" will block when the + queue reaches maxsize, until an item is removed by get(). + + Unlike the standard library Queue, you can reliably know this Queue's size + with qsize(), since your single-threaded asyncio application won't be + interrupted between calling qsize() and doing an operation on the Queue. + """ + + def __init__(self, maxsize=0, *, loop=None): + if loop is None: + self._loop = events.get_event_loop() + else: + self._loop = loop + self._maxsize = maxsize + + # Futures. + self._getters = collections.deque() + # Futures. + self._putters = collections.deque() + self._unfinished_tasks = 0 + self._finished = locks.Event(loop=self._loop) + self._finished.set() + self._init(maxsize) + + # These three are overridable in subclasses. + + def _init(self, maxsize): + self._queue = collections.deque() + + def _get(self): + return self._queue.popleft() + + def _put(self, item): + self._queue.append(item) + + # End of the overridable methods. + + def _wakeup_next(self, waiters): + # Wake up the next waiter (if any) that isn't cancelled. + while waiters: + waiter = waiters.popleft() + if not waiter.done(): + waiter.set_result(None) + break + + def __repr__(self): + return '<{} at {:#x} {}>'.format( + type(self).__name__, id(self), self._format()) + + def __str__(self): + return '<{} {}>'.format(type(self).__name__, self._format()) + + def _format(self): + result = 'maxsize={!r}'.format(self._maxsize) + if getattr(self, '_queue', None): + result += ' _queue={!r}'.format(list(self._queue)) + if self._getters: + result += ' _getters[{}]'.format(len(self._getters)) + if self._putters: + result += ' _putters[{}]'.format(len(self._putters)) + if self._unfinished_tasks: + result += ' tasks={}'.format(self._unfinished_tasks) + return result + + def qsize(self): + """Number of items in the queue.""" + return len(self._queue) + + @property + def maxsize(self): + """Number of items allowed in the queue.""" + return self._maxsize + + def empty(self): + """Return True if the queue is empty, False otherwise.""" + return not self._queue + + def full(self): + """Return True if there are maxsize items in the queue. + + Note: if the Queue was initialized with maxsize=0 (the default), + then full() is never True. + """ + if self._maxsize <= 0: + return False + else: + return self.qsize() >= self._maxsize + + @coroutine + def put(self, item): + """Put an item into the queue. + + Put an item into the queue. If the queue is full, wait until a free + slot is available before adding item. + + This method is a coroutine. + """ + while self.full(): + putter = self._loop.create_future() + self._putters.append(putter) + try: + yield from putter + except: + putter.cancel() # Just in case putter is not done yet. + if not self.full() and not putter.cancelled(): + # We were woken up by get_nowait(), but can't take + # the call. Wake up the next in line. + self._wakeup_next(self._putters) + raise + return self.put_nowait(item) + + def put_nowait(self, item): + """Put an item into the queue without blocking. + + If no free slot is immediately available, raise QueueFull. + """ + if self.full(): + raise QueueFull + self._put(item) + self._unfinished_tasks += 1 + self._finished.clear() + self._wakeup_next(self._getters) + + @coroutine + def get(self): + """Remove and return an item from the queue. + + If queue is empty, wait until an item is available. + + This method is a coroutine. + """ + while self.empty(): + getter = self._loop.create_future() + self._getters.append(getter) + try: + yield from getter + except: + getter.cancel() # Just in case getter is not done yet. + if not self.empty() and not getter.cancelled(): + # We were woken up by put_nowait(), but can't take + # the call. Wake up the next in line. + self._wakeup_next(self._getters) + raise + return self.get_nowait() + + def get_nowait(self): + """Remove and return an item from the queue. + + Return an item if one is immediately available, else raise QueueEmpty. + """ + if self.empty(): + raise QueueEmpty + item = self._get() + self._wakeup_next(self._putters) + return item + + def task_done(self): + """Indicate that a formerly enqueued task is complete. + + Used by queue consumers. For each get() used to fetch a task, + a subsequent call to task_done() tells the queue that the processing + on the task is complete. + + If a join() is currently blocking, it will resume when all items have + been processed (meaning that a task_done() call was received for every + item that had been put() into the queue). + + Raises ValueError if called more times than there were items placed in + the queue. + """ + if self._unfinished_tasks <= 0: + raise ValueError('task_done() called too many times') + self._unfinished_tasks -= 1 + if self._unfinished_tasks == 0: + self._finished.set() + + @coroutine + def join(self): + """Block until all items in the queue have been gotten and processed. + + The count of unfinished tasks goes up whenever an item is added to the + queue. The count goes down whenever a consumer calls task_done() to + indicate that the item was retrieved and all work on it is complete. + When the count of unfinished tasks drops to zero, join() unblocks. + """ + if self._unfinished_tasks > 0: + yield from self._finished.wait() + + +class PriorityQueue(Queue): + """A subclass of Queue; retrieves entries in priority order (lowest first). + + Entries are typically tuples of the form: (priority number, data). + """ + + def _init(self, maxsize): + self._queue = [] + + def _put(self, item, heappush=heapq.heappush): + heappush(self._queue, item) + + def _get(self, heappop=heapq.heappop): + return heappop(self._queue) + + +class LifoQueue(Queue): + """A subclass of Queue that retrieves most recently added entries first.""" + + def _init(self, maxsize): + self._queue = [] + + def _put(self, item): + self._queue.append(item) + + def _get(self): + return self._queue.pop() + + +if not compat.PY35: + JoinableQueue = Queue + """Deprecated alias for Queue.""" + __all__.append('JoinableQueue') diff --git a/thirdparty/asyncio/asyncio/selector_events.py b/thirdparty/asyncio/asyncio/selector_events.py new file mode 100644 index 0000000..12d357b --- /dev/null +++ b/thirdparty/asyncio/asyncio/selector_events.py @@ -0,0 +1,1141 @@ +"""Event loop using a selector and related classes. + +A selector is a "notify-when-ready" multiplexer. For a subclass which +also includes support for signal handling, see the unix_events sub-module. +""" + +__all__ = ['BaseSelectorEventLoop'] + +import collections +import errno +import functools +import socket +import warnings +import weakref +try: + import ssl +except ImportError: # pragma: no cover + ssl = None + +from . import base_events +from . import compat +from . import constants +from . import events +from . import futures +from . import selectors +from . import transports +from . import sslproto +from .coroutines import coroutine +from .log import logger + + +def _test_selector_event(selector, fd, event): + # Test if the selector is monitoring 'event' events + # for the file descriptor 'fd'. + try: + key = selector.get_key(fd) + except KeyError: + return False + else: + return bool(key.events & event) + + +if hasattr(socket, 'TCP_NODELAY'): + def _set_nodelay(sock): + if (sock.family in {socket.AF_INET, socket.AF_INET6} and + sock.type == socket.SOCK_STREAM and + sock.proto == socket.IPPROTO_TCP): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) +else: + def _set_nodelay(sock): + pass + + +class BaseSelectorEventLoop(base_events.BaseEventLoop): + """Selector event loop. + + See events.EventLoop for API specification. + """ + + def __init__(self, selector=None): + super().__init__() + + if selector is None: + selector = selectors.DefaultSelector() + logger.debug('Using selector: %s', selector.__class__.__name__) + self._selector = selector + self._make_self_pipe() + self._transports = weakref.WeakValueDictionary() + + def _make_socket_transport(self, sock, protocol, waiter=None, *, + extra=None, server=None): + return _SelectorSocketTransport(self, sock, protocol, waiter, + extra, server) + + def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None): + if not sslproto._is_sslproto_available(): + return self._make_legacy_ssl_transport( + rawsock, protocol, sslcontext, waiter, + server_side=server_side, server_hostname=server_hostname, + extra=extra, server=server) + + ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, + server_side, server_hostname) + _SelectorSocketTransport(self, rawsock, ssl_protocol, + extra=extra, server=server) + return ssl_protocol._app_transport + + def _make_legacy_ssl_transport(self, rawsock, protocol, sslcontext, + waiter, *, + server_side=False, server_hostname=None, + extra=None, server=None): + # Use the legacy API: SSL_write, SSL_read, etc. The legacy API is used + # on Python 3.4 and older, when ssl.MemoryBIO is not available. + return _SelectorSslTransport( + self, rawsock, protocol, sslcontext, waiter, + server_side, server_hostname, extra, server) + + def _make_datagram_transport(self, sock, protocol, + address=None, waiter=None, extra=None): + return _SelectorDatagramTransport(self, sock, protocol, + address, waiter, extra) + + def close(self): + if self.is_running(): + raise RuntimeError("Cannot close a running event loop") + if self.is_closed(): + return + self._close_self_pipe() + super().close() + if self._selector is not None: + self._selector.close() + self._selector = None + + def _socketpair(self): + raise NotImplementedError + + def _close_self_pipe(self): + self._remove_reader(self._ssock.fileno()) + self._ssock.close() + self._ssock = None + self._csock.close() + self._csock = None + self._internal_fds -= 1 + + def _make_self_pipe(self): + # A self-socket, really. :-) + self._ssock, self._csock = self._socketpair() + self._ssock.setblocking(False) + self._csock.setblocking(False) + self._internal_fds += 1 + self._add_reader(self._ssock.fileno(), self._read_from_self) + + def _process_self_data(self, data): + pass + + def _read_from_self(self): + while True: + try: + data = self._ssock.recv(4096) + if not data: + break + self._process_self_data(data) + except InterruptedError: + continue + except BlockingIOError: + break + + def _write_to_self(self): + # This may be called from a different thread, possibly after + # _close_self_pipe() has been called or even while it is + # running. Guard for self._csock being None or closed. When + # a socket is closed, send() raises OSError (with errno set to + # EBADF, but let's not rely on the exact error code). + csock = self._csock + if csock is not None: + try: + csock.send(b'\0') + except OSError: + if self._debug: + logger.debug("Fail to write a null byte into the " + "self-pipe socket", + exc_info=True) + + def _start_serving(self, protocol_factory, sock, + sslcontext=None, server=None, backlog=100): + self._add_reader(sock.fileno(), self._accept_connection, + protocol_factory, sock, sslcontext, server, backlog) + + def _accept_connection(self, protocol_factory, sock, + sslcontext=None, server=None, backlog=100): + # This method is only called once for each event loop tick where the + # listening socket has triggered an EVENT_READ. There may be multiple + # connections waiting for an .accept() so it is called in a loop. + # See https://bugs.python.org/issue27906 for more details. + for _ in range(backlog): + try: + conn, addr = sock.accept() + if self._debug: + logger.debug("%r got a new connection from %r: %r", + server, addr, conn) + conn.setblocking(False) + except (BlockingIOError, InterruptedError, ConnectionAbortedError): + # Early exit because the socket accept buffer is empty. + return None + except OSError as exc: + # There's nowhere to send the error, so just log it. + if exc.errno in (errno.EMFILE, errno.ENFILE, + errno.ENOBUFS, errno.ENOMEM): + # Some platforms (e.g. Linux keep reporting the FD as + # ready, so we remove the read handler temporarily. + # We'll try again in a while. + self.call_exception_handler({ + 'message': 'socket.accept() out of system resource', + 'exception': exc, + 'socket': sock, + }) + self._remove_reader(sock.fileno()) + self.call_later(constants.ACCEPT_RETRY_DELAY, + self._start_serving, + protocol_factory, sock, sslcontext, server, + backlog) + else: + raise # The event loop will catch, log and ignore it. + else: + extra = {'peername': addr} + accept = self._accept_connection2(protocol_factory, conn, extra, + sslcontext, server) + self.create_task(accept) + + @coroutine + def _accept_connection2(self, protocol_factory, conn, extra, + sslcontext=None, server=None): + protocol = None + transport = None + try: + protocol = protocol_factory() + waiter = self.create_future() + if sslcontext: + transport = self._make_ssl_transport( + conn, protocol, sslcontext, waiter=waiter, + server_side=True, extra=extra, server=server) + else: + transport = self._make_socket_transport( + conn, protocol, waiter=waiter, extra=extra, + server=server) + + try: + yield from waiter + except: + transport.close() + raise + + # It's now up to the protocol to handle the connection. + except Exception as exc: + if self._debug: + context = { + 'message': ('Error on transport creation ' + 'for incoming connection'), + 'exception': exc, + } + if protocol is not None: + context['protocol'] = protocol + if transport is not None: + context['transport'] = transport + self.call_exception_handler(context) + + def _ensure_fd_no_transport(self, fd): + try: + transport = self._transports[fd] + except KeyError: + pass + else: + if not transport.is_closing(): + raise RuntimeError( + 'File descriptor {!r} is used by transport {!r}'.format( + fd, transport)) + + def _add_reader(self, fd, callback, *args): + self._check_closed() + handle = events.Handle(callback, args, self) + try: + key = self._selector.get_key(fd) + except KeyError: + self._selector.register(fd, selectors.EVENT_READ, + (handle, None)) + else: + mask, (reader, writer) = key.events, key.data + self._selector.modify(fd, mask | selectors.EVENT_READ, + (handle, writer)) + if reader is not None: + reader.cancel() + + def _remove_reader(self, fd): + if self.is_closed(): + return False + try: + key = self._selector.get_key(fd) + except KeyError: + return False + else: + mask, (reader, writer) = key.events, key.data + mask &= ~selectors.EVENT_READ + if not mask: + self._selector.unregister(fd) + else: + self._selector.modify(fd, mask, (None, writer)) + + if reader is not None: + reader.cancel() + return True + else: + return False + + def _add_writer(self, fd, callback, *args): + self._check_closed() + handle = events.Handle(callback, args, self) + try: + key = self._selector.get_key(fd) + except KeyError: + self._selector.register(fd, selectors.EVENT_WRITE, + (None, handle)) + else: + mask, (reader, writer) = key.events, key.data + self._selector.modify(fd, mask | selectors.EVENT_WRITE, + (reader, handle)) + if writer is not None: + writer.cancel() + + def _remove_writer(self, fd): + """Remove a writer callback.""" + if self.is_closed(): + return False + try: + key = self._selector.get_key(fd) + except KeyError: + return False + else: + mask, (reader, writer) = key.events, key.data + # Remove both writer and connector. + mask &= ~selectors.EVENT_WRITE + if not mask: + self._selector.unregister(fd) + else: + self._selector.modify(fd, mask, (reader, None)) + + if writer is not None: + writer.cancel() + return True + else: + return False + + def add_reader(self, fd, callback, *args): + """Add a reader callback.""" + self._ensure_fd_no_transport(fd) + return self._add_reader(fd, callback, *args) + + def remove_reader(self, fd): + """Remove a reader callback.""" + self._ensure_fd_no_transport(fd) + return self._remove_reader(fd) + + def add_writer(self, fd, callback, *args): + """Add a writer callback..""" + self._ensure_fd_no_transport(fd) + return self._add_writer(fd, callback, *args) + + def remove_writer(self, fd): + """Remove a writer callback.""" + self._ensure_fd_no_transport(fd) + return self._remove_writer(fd) + + def sock_recv(self, sock, n): + """Receive data from the socket. + + The return value is a bytes object representing the data received. + The maximum amount of data to be received at once is specified by + nbytes. + + This method is a coroutine. + """ + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + fut = self.create_future() + self._sock_recv(fut, False, sock, n) + return fut + + def _sock_recv(self, fut, registered, sock, n): + # _sock_recv() can add itself as an I/O callback if the operation can't + # be done immediately. Don't use it directly, call sock_recv(). + fd = sock.fileno() + if registered: + # Remove the callback early. It should be rare that the + # selector says the fd is ready but the call still returns + # EAGAIN, and I am willing to take a hit in that case in + # order to simplify the common case. + self.remove_reader(fd) + if fut.cancelled(): + return + try: + data = sock.recv(n) + except (BlockingIOError, InterruptedError): + self.add_reader(fd, self._sock_recv, fut, True, sock, n) + except Exception as exc: + fut.set_exception(exc) + else: + fut.set_result(data) + + def sock_sendall(self, sock, data): + """Send data to the socket. + + The socket must be connected to a remote socket. This method continues + to send data from data until either all data has been sent or an + error occurs. None is returned on success. On error, an exception is + raised, and there is no way to determine how much data, if any, was + successfully processed by the receiving end of the connection. + + This method is a coroutine. + """ + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + fut = self.create_future() + if data: + self._sock_sendall(fut, False, sock, data) + else: + fut.set_result(None) + return fut + + def _sock_sendall(self, fut, registered, sock, data): + fd = sock.fileno() + + if registered: + self.remove_writer(fd) + if fut.cancelled(): + return + + try: + n = sock.send(data) + except (BlockingIOError, InterruptedError): + n = 0 + except Exception as exc: + fut.set_exception(exc) + return + + if n == len(data): + fut.set_result(None) + else: + if n: + data = data[n:] + self.add_writer(fd, self._sock_sendall, fut, True, sock, data) + + @coroutine + def sock_connect(self, sock, address): + """Connect to a remote socket at address. + + This method is a coroutine. + """ + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + + if not hasattr(socket, 'AF_UNIX') or sock.family != socket.AF_UNIX: + resolved = base_events._ensure_resolved( + address, family=sock.family, proto=sock.proto, loop=self) + if not resolved.done(): + yield from resolved + _, _, _, _, address = resolved.result()[0] + + fut = self.create_future() + self._sock_connect(fut, sock, address) + return (yield from fut) + + def _sock_connect(self, fut, sock, address): + fd = sock.fileno() + try: + sock.connect(address) + except (BlockingIOError, InterruptedError): + # Issue #23618: When the C function connect() fails with EINTR, the + # connection runs in background. We have to wait until the socket + # becomes writable to be notified when the connection succeed or + # fails. + fut.add_done_callback( + functools.partial(self._sock_connect_done, fd)) + self.add_writer(fd, self._sock_connect_cb, fut, sock, address) + except Exception as exc: + fut.set_exception(exc) + else: + fut.set_result(None) + + def _sock_connect_done(self, fd, fut): + self.remove_writer(fd) + + def _sock_connect_cb(self, fut, sock, address): + if fut.cancelled(): + return + + try: + err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err != 0: + # Jump to any except clause below. + raise OSError(err, 'Connect call failed %s' % (address,)) + except (BlockingIOError, InterruptedError): + # socket is still registered, the callback will be retried later + pass + except Exception as exc: + fut.set_exception(exc) + else: + fut.set_result(None) + + def sock_accept(self, sock): + """Accept a connection. + + The socket must be bound to an address and listening for connections. + The return value is a pair (conn, address) where conn is a new socket + object usable to send and receive data on the connection, and address + is the address bound to the socket on the other end of the connection. + + This method is a coroutine. + """ + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + fut = self.create_future() + self._sock_accept(fut, False, sock) + return fut + + def _sock_accept(self, fut, registered, sock): + fd = sock.fileno() + if registered: + self.remove_reader(fd) + if fut.cancelled(): + return + try: + conn, address = sock.accept() + conn.setblocking(False) + except (BlockingIOError, InterruptedError): + self.add_reader(fd, self._sock_accept, fut, True, sock) + except Exception as exc: + fut.set_exception(exc) + else: + fut.set_result((conn, address)) + + def _process_events(self, event_list): + for key, mask in event_list: + fileobj, (reader, writer) = key.fileobj, key.data + if mask & selectors.EVENT_READ and reader is not None: + if reader._cancelled: + self._remove_reader(fileobj) + else: + self._add_callback(reader) + if mask & selectors.EVENT_WRITE and writer is not None: + if writer._cancelled: + self._remove_writer(fileobj) + else: + self._add_callback(writer) + + def _stop_serving(self, sock): + self._remove_reader(sock.fileno()) + sock.close() + + +class _SelectorTransport(transports._FlowControlMixin, + transports.Transport): + + max_size = 256 * 1024 # Buffer size passed to recv(). + + _buffer_factory = bytearray # Constructs initial value for self._buffer. + + # Attribute used in the destructor: it must be set even if the constructor + # is not called (see _SelectorSslTransport which may start by raising an + # exception) + _sock = None + + def __init__(self, loop, sock, protocol, extra=None, server=None): + super().__init__(extra, loop) + self._extra['socket'] = sock + self._extra['sockname'] = sock.getsockname() + if 'peername' not in self._extra: + try: + self._extra['peername'] = sock.getpeername() + except socket.error: + self._extra['peername'] = None + self._sock = sock + self._sock_fd = sock.fileno() + self._protocol = protocol + self._protocol_connected = True + self._server = server + self._buffer = self._buffer_factory() + self._conn_lost = 0 # Set when call to connection_lost scheduled. + self._closing = False # Set when close() called. + if self._server is not None: + self._server._attach() + loop._transports[self._sock_fd] = self + + def __repr__(self): + info = [self.__class__.__name__] + if self._sock is None: + info.append('closed') + elif self._closing: + info.append('closing') + info.append('fd=%s' % self._sock_fd) + # test if the transport was closed + if self._loop is not None and not self._loop.is_closed(): + polling = _test_selector_event(self._loop._selector, + self._sock_fd, selectors.EVENT_READ) + if polling: + info.append('read=polling') + else: + info.append('read=idle') + + polling = _test_selector_event(self._loop._selector, + self._sock_fd, + selectors.EVENT_WRITE) + if polling: + state = 'polling' + else: + state = 'idle' + + bufsize = self.get_write_buffer_size() + info.append('write=<%s, bufsize=%s>' % (state, bufsize)) + return '<%s>' % ' '.join(info) + + def abort(self): + self._force_close(None) + + def set_protocol(self, protocol): + self._protocol = protocol + + def get_protocol(self): + return self._protocol + + def is_closing(self): + return self._closing + + def close(self): + if self._closing: + return + self._closing = True + self._loop._remove_reader(self._sock_fd) + if not self._buffer: + self._conn_lost += 1 + self._loop._remove_writer(self._sock_fd) + self._loop.call_soon(self._call_connection_lost, None) + + # On Python 3.3 and older, objects with a destructor part of a reference + # cycle are never destroyed. It's not more the case on Python 3.4 thanks + # to the PEP 442. + if compat.PY34: + def __del__(self): + if self._sock is not None: + warnings.warn("unclosed transport %r" % self, ResourceWarning) + self._sock.close() + + def _fatal_error(self, exc, message='Fatal error on transport'): + # Should be called from exception handler only. + if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if self._loop.get_debug(): + logger.debug("%r: %s", self, message, exc_info=True) + else: + self._loop.call_exception_handler({ + 'message': message, + 'exception': exc, + 'transport': self, + 'protocol': self._protocol, + }) + self._force_close(exc) + + def _force_close(self, exc): + if self._conn_lost: + return + if self._buffer: + self._buffer.clear() + self._loop._remove_writer(self._sock_fd) + if not self._closing: + self._closing = True + self._loop._remove_reader(self._sock_fd) + self._conn_lost += 1 + self._loop.call_soon(self._call_connection_lost, exc) + + def _call_connection_lost(self, exc): + try: + if self._protocol_connected: + self._protocol.connection_lost(exc) + finally: + self._sock.close() + self._sock = None + self._protocol = None + self._loop = None + server = self._server + if server is not None: + server._detach() + self._server = None + + def get_write_buffer_size(self): + return len(self._buffer) + + +class _SelectorSocketTransport(_SelectorTransport): + + def __init__(self, loop, sock, protocol, waiter=None, + extra=None, server=None): + super().__init__(loop, sock, protocol, extra, server) + self._eof = False + self._paused = False + + # Disable the Nagle algorithm -- small writes will be + # sent without waiting for the TCP ACK. This generally + # decreases the latency (in some cases significantly.) + _set_nodelay(self._sock) + + self._loop.call_soon(self._protocol.connection_made, self) + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop._add_reader, + self._sock_fd, self._read_ready) + if waiter is not None: + # only wake up the waiter when connection_made() has been called + self._loop.call_soon(futures._set_result_unless_cancelled, + waiter, None) + + def pause_reading(self): + if self._closing: + raise RuntimeError('Cannot pause_reading() when closing') + if self._paused: + raise RuntimeError('Already paused') + self._paused = True + self._loop._remove_reader(self._sock_fd) + if self._loop.get_debug(): + logger.debug("%r pauses reading", self) + + def resume_reading(self): + if not self._paused: + raise RuntimeError('Not paused') + self._paused = False + if self._closing: + return + self._loop._add_reader(self._sock_fd, self._read_ready) + if self._loop.get_debug(): + logger.debug("%r resumes reading", self) + + def _read_ready(self): + if self._conn_lost: + return + try: + data = self._sock.recv(self.max_size) + except (BlockingIOError, InterruptedError): + pass + except Exception as exc: + self._fatal_error(exc, 'Fatal read error on socket transport') + else: + if data: + self._protocol.data_received(data) + else: + if self._loop.get_debug(): + logger.debug("%r received EOF", self) + keep_open = self._protocol.eof_received() + if keep_open: + # We're keeping the connection open so the + # protocol can write more, but we still can't + # receive more, so remove the reader callback. + self._loop._remove_reader(self._sock_fd) + else: + self.close() + + def write(self, data): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be a bytes-like object, ' + 'not %r' % type(data).__name__) + if self._eof: + raise RuntimeError('Cannot call write() after write_eof()') + if not data: + return + + if self._conn_lost: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.send() raised exception.') + self._conn_lost += 1 + return + + if not self._buffer: + # Optimization: try to send now. + try: + n = self._sock.send(data) + except (BlockingIOError, InterruptedError): + pass + except Exception as exc: + self._fatal_error(exc, 'Fatal write error on socket transport') + return + else: + data = data[n:] + if not data: + return + # Not all was written; register write handler. + self._loop._add_writer(self._sock_fd, self._write_ready) + + # Add it to the buffer. + self._buffer.extend(data) + self._maybe_pause_protocol() + + def _write_ready(self): + assert self._buffer, 'Data should not be empty' + + if self._conn_lost: + return + try: + n = self._sock.send(self._buffer) + except (BlockingIOError, InterruptedError): + pass + except Exception as exc: + self._loop._remove_writer(self._sock_fd) + self._buffer.clear() + self._fatal_error(exc, 'Fatal write error on socket transport') + else: + if n: + del self._buffer[:n] + self._maybe_resume_protocol() # May append to buffer. + if not self._buffer: + self._loop._remove_writer(self._sock_fd) + if self._closing: + self._call_connection_lost(None) + elif self._eof: + self._sock.shutdown(socket.SHUT_WR) + + def write_eof(self): + if self._eof: + return + self._eof = True + if not self._buffer: + self._sock.shutdown(socket.SHUT_WR) + + def can_write_eof(self): + return True + + +class _SelectorSslTransport(_SelectorTransport): + + _buffer_factory = bytearray + + def __init__(self, loop, rawsock, protocol, sslcontext, waiter=None, + server_side=False, server_hostname=None, + extra=None, server=None): + if ssl is None: + raise RuntimeError('stdlib ssl module not available') + + if not sslcontext: + sslcontext = sslproto._create_transport_context(server_side, server_hostname) + + wrap_kwargs = { + 'server_side': server_side, + 'do_handshake_on_connect': False, + } + if server_hostname and not server_side: + wrap_kwargs['server_hostname'] = server_hostname + sslsock = sslcontext.wrap_socket(rawsock, **wrap_kwargs) + + super().__init__(loop, sslsock, protocol, extra, server) + # the protocol connection is only made after the SSL handshake + self._protocol_connected = False + + self._server_hostname = server_hostname + self._waiter = waiter + self._sslcontext = sslcontext + self._paused = False + + # SSL-specific extra info. (peercert is set later) + self._extra.update(sslcontext=sslcontext) + + if self._loop.get_debug(): + logger.debug("%r starts SSL handshake", self) + start_time = self._loop.time() + else: + start_time = None + self._on_handshake(start_time) + + def _wakeup_waiter(self, exc=None): + if self._waiter is None: + return + if not self._waiter.cancelled(): + if exc is not None: + self._waiter.set_exception(exc) + else: + self._waiter.set_result(None) + self._waiter = None + + def _on_handshake(self, start_time): + try: + self._sock.do_handshake() + except ssl.SSLWantReadError: + self._loop._add_reader(self._sock_fd, + self._on_handshake, start_time) + return + except ssl.SSLWantWriteError: + self._loop._add_writer(self._sock_fd, + self._on_handshake, start_time) + return + except BaseException as exc: + if self._loop.get_debug(): + logger.warning("%r: SSL handshake failed", + self, exc_info=True) + self._loop._remove_reader(self._sock_fd) + self._loop._remove_writer(self._sock_fd) + self._sock.close() + self._wakeup_waiter(exc) + if isinstance(exc, Exception): + return + else: + raise + + self._loop._remove_reader(self._sock_fd) + self._loop._remove_writer(self._sock_fd) + + peercert = self._sock.getpeercert() + if not hasattr(self._sslcontext, 'check_hostname'): + # Verify hostname if requested, Python 3.4+ uses check_hostname + # and checks the hostname in do_handshake() + if (self._server_hostname and + self._sslcontext.verify_mode != ssl.CERT_NONE): + try: + ssl.match_hostname(peercert, self._server_hostname) + except Exception as exc: + if self._loop.get_debug(): + logger.warning("%r: SSL handshake failed " + "on matching the hostname", + self, exc_info=True) + self._sock.close() + self._wakeup_waiter(exc) + return + + # Add extra info that becomes available after handshake. + self._extra.update(peercert=peercert, + cipher=self._sock.cipher(), + compression=self._sock.compression(), + ssl_object=self._sock, + ) + + self._read_wants_write = False + self._write_wants_read = False + self._loop._add_reader(self._sock_fd, self._read_ready) + self._protocol_connected = True + self._loop.call_soon(self._protocol.connection_made, self) + # only wake up the waiter when connection_made() has been called + self._loop.call_soon(self._wakeup_waiter) + + if self._loop.get_debug(): + dt = self._loop.time() - start_time + logger.debug("%r: SSL handshake took %.1f ms", self, dt * 1e3) + + def pause_reading(self): + # XXX This is a bit icky, given the comment at the top of + # _read_ready(). Is it possible to evoke a deadlock? I don't + # know, although it doesn't look like it; write() will still + # accept more data for the buffer and eventually the app will + # call resume_reading() again, and things will flow again. + + if self._closing: + raise RuntimeError('Cannot pause_reading() when closing') + if self._paused: + raise RuntimeError('Already paused') + self._paused = True + self._loop._remove_reader(self._sock_fd) + if self._loop.get_debug(): + logger.debug("%r pauses reading", self) + + def resume_reading(self): + if not self._paused: + raise RuntimeError('Not paused') + self._paused = False + if self._closing: + return + self._loop._add_reader(self._sock_fd, self._read_ready) + if self._loop.get_debug(): + logger.debug("%r resumes reading", self) + + def _read_ready(self): + if self._conn_lost: + return + if self._write_wants_read: + self._write_wants_read = False + self._write_ready() + + if self._buffer: + self._loop._add_writer(self._sock_fd, self._write_ready) + + try: + data = self._sock.recv(self.max_size) + except (BlockingIOError, InterruptedError, ssl.SSLWantReadError): + pass + except ssl.SSLWantWriteError: + self._read_wants_write = True + self._loop._remove_reader(self._sock_fd) + self._loop._add_writer(self._sock_fd, self._write_ready) + except Exception as exc: + self._fatal_error(exc, 'Fatal read error on SSL transport') + else: + if data: + self._protocol.data_received(data) + else: + try: + if self._loop.get_debug(): + logger.debug("%r received EOF", self) + keep_open = self._protocol.eof_received() + if keep_open: + logger.warning('returning true from eof_received() ' + 'has no effect when using ssl') + finally: + self.close() + + def _write_ready(self): + if self._conn_lost: + return + if self._read_wants_write: + self._read_wants_write = False + self._read_ready() + + if not (self._paused or self._closing): + self._loop._add_reader(self._sock_fd, self._read_ready) + + if self._buffer: + try: + n = self._sock.send(self._buffer) + except (BlockingIOError, InterruptedError, ssl.SSLWantWriteError): + n = 0 + except ssl.SSLWantReadError: + n = 0 + self._loop._remove_writer(self._sock_fd) + self._write_wants_read = True + except Exception as exc: + self._loop._remove_writer(self._sock_fd) + self._buffer.clear() + self._fatal_error(exc, 'Fatal write error on SSL transport') + return + + if n: + del self._buffer[:n] + + self._maybe_resume_protocol() # May append to buffer. + + if not self._buffer: + self._loop._remove_writer(self._sock_fd) + if self._closing: + self._call_connection_lost(None) + + def write(self, data): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be a bytes-like object, ' + 'not %r' % type(data).__name__) + if not data: + return + + if self._conn_lost: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.send() raised exception.') + self._conn_lost += 1 + return + + if not self._buffer: + self._loop._add_writer(self._sock_fd, self._write_ready) + + # Add it to the buffer. + self._buffer.extend(data) + self._maybe_pause_protocol() + + def can_write_eof(self): + return False + + +class _SelectorDatagramTransport(_SelectorTransport): + + _buffer_factory = collections.deque + + def __init__(self, loop, sock, protocol, address=None, + waiter=None, extra=None): + super().__init__(loop, sock, protocol, extra) + self._address = address + self._loop.call_soon(self._protocol.connection_made, self) + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop._add_reader, + self._sock_fd, self._read_ready) + if waiter is not None: + # only wake up the waiter when connection_made() has been called + self._loop.call_soon(futures._set_result_unless_cancelled, + waiter, None) + + def get_write_buffer_size(self): + return sum(len(data) for data, _ in self._buffer) + + def _read_ready(self): + if self._conn_lost: + return + try: + data, addr = self._sock.recvfrom(self.max_size) + except (BlockingIOError, InterruptedError): + pass + except OSError as exc: + self._protocol.error_received(exc) + except Exception as exc: + self._fatal_error(exc, 'Fatal read error on datagram transport') + else: + self._protocol.datagram_received(data, addr) + + def sendto(self, data, addr=None): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be a bytes-like object, ' + 'not %r' % type(data).__name__) + if not data: + return + + if self._address and addr not in (None, self._address): + raise ValueError('Invalid address: must be None or %s' % + (self._address,)) + + if self._conn_lost and self._address: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.send() raised exception.') + self._conn_lost += 1 + return + + if not self._buffer: + # Attempt to send it right away first. + try: + if self._address: + self._sock.send(data) + else: + self._sock.sendto(data, addr) + return + except (BlockingIOError, InterruptedError): + self._loop._add_writer(self._sock_fd, self._sendto_ready) + except OSError as exc: + self._protocol.error_received(exc) + return + except Exception as exc: + self._fatal_error(exc, + 'Fatal write error on datagram transport') + return + + # Ensure that what we buffer is immutable. + self._buffer.append((bytes(data), addr)) + self._maybe_pause_protocol() + + def _sendto_ready(self): + while self._buffer: + data, addr = self._buffer.popleft() + try: + if self._address: + self._sock.send(data) + else: + self._sock.sendto(data, addr) + except (BlockingIOError, InterruptedError): + self._buffer.appendleft((data, addr)) # Try again later. + break + except OSError as exc: + self._protocol.error_received(exc) + return + except Exception as exc: + self._fatal_error(exc, + 'Fatal write error on datagram transport') + return + + self._maybe_resume_protocol() # May append to buffer. + if not self._buffer: + self._loop._remove_writer(self._sock_fd) + if self._closing: + self._call_connection_lost(None) diff --git a/thirdparty/asyncio/asyncio/selectors.py b/thirdparty/asyncio/asyncio/selectors.py new file mode 100644 index 0000000..89680a2 --- /dev/null +++ b/thirdparty/asyncio/asyncio/selectors.py @@ -0,0 +1,611 @@ +"""Selectors module. + +This module allows high-level and efficient I/O multiplexing, built upon the +`select` module primitives. +""" + + +from abc import ABCMeta, abstractmethod +from collections import namedtuple, Mapping +import math +import select +import sys + + +# generic events, that must be mapped to implementation-specific ones +EVENT_READ = (1 << 0) +EVENT_WRITE = (1 << 1) + + +def _fileobj_to_fd(fileobj): + """Return a file descriptor from a file object. + + Parameters: + fileobj -- file object or file descriptor + + Returns: + corresponding file descriptor + + Raises: + ValueError if the object is invalid + """ + if isinstance(fileobj, int): + fd = fileobj + else: + try: + fd = int(fileobj.fileno()) + except (AttributeError, TypeError, ValueError): + raise ValueError("Invalid file object: " + "{!r}".format(fileobj)) from None + if fd < 0: + raise ValueError("Invalid file descriptor: {}".format(fd)) + return fd + + +SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) + +SelectorKey.__doc__ = """SelectorKey(fileobj, fd, events, data) + + Object used to associate a file object to its backing + file descriptor, selected event mask, and attached data. +""" +if sys.version_info >= (3, 5): + SelectorKey.fileobj.__doc__ = 'File object registered.' + SelectorKey.fd.__doc__ = 'Underlying file descriptor.' + SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' + SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. + For example, this could be used to store a per-client session ID.''') + +class _SelectorMapping(Mapping): + """Mapping of file objects to selector keys.""" + + def __init__(self, selector): + self._selector = selector + + def __len__(self): + return len(self._selector._fd_to_key) + + def __getitem__(self, fileobj): + try: + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key[fd] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + + def __iter__(self): + return iter(self._selector._fd_to_key) + + +class BaseSelector(metaclass=ABCMeta): + """Selector abstract base class. + + A selector supports registering file objects to be monitored for specific + I/O events. + + A file object is a file descriptor or any object with a `fileno()` method. + An arbitrary object can be attached to the file object, which can be used + for example to store context information, a callback, etc. + + A selector can use various implementations (select(), poll(), epoll()...) + depending on the platform. The default `Selector` class uses the most + efficient implementation on the current platform. + """ + + @abstractmethod + def register(self, fileobj, events, data=None): + """Register a file object. + + Parameters: + fileobj -- file object or file descriptor + events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) + data -- attached data + + Returns: + SelectorKey instance + + Raises: + ValueError if events is invalid + KeyError if fileobj is already registered + OSError if fileobj is closed or otherwise is unacceptable to + the underlying system call (if a system call is made) + + Note: + OSError may or may not be raised + """ + raise NotImplementedError + + @abstractmethod + def unregister(self, fileobj): + """Unregister a file object. + + Parameters: + fileobj -- file object or file descriptor + + Returns: + SelectorKey instance + + Raises: + KeyError if fileobj is not registered + + Note: + If fileobj is registered but has since been closed this does + *not* raise OSError (even if the wrapped syscall does) + """ + raise NotImplementedError + + def modify(self, fileobj, events, data=None): + """Change a registered file object monitored events or attached data. + + Parameters: + fileobj -- file object or file descriptor + events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) + data -- attached data + + Returns: + SelectorKey instance + + Raises: + Anything that unregister() or register() raises + """ + self.unregister(fileobj) + return self.register(fileobj, events, data) + + @abstractmethod + def select(self, timeout=None): + """Perform the actual selection, until some monitored file objects are + ready or a timeout expires. + + Parameters: + timeout -- if timeout > 0, this specifies the maximum wait time, in + seconds + if timeout <= 0, the select() call won't block, and will + report the currently ready file objects + if timeout is None, select() will block until a monitored + file object becomes ready + + Returns: + list of (key, events) for ready file objects + `events` is a bitwise mask of EVENT_READ|EVENT_WRITE + """ + raise NotImplementedError + + def close(self): + """Close the selector. + + This must be called to make sure that any underlying resource is freed. + """ + pass + + def get_key(self, fileobj): + """Return the key associated to a registered file object. + + Returns: + SelectorKey for this file object + """ + mapping = self.get_map() + if mapping is None: + raise RuntimeError('Selector is closed') + try: + return mapping[fileobj] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + + @abstractmethod + def get_map(self): + """Return a mapping of file objects to selector keys.""" + raise NotImplementedError + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +class _BaseSelectorImpl(BaseSelector): + """Base selector implementation.""" + + def __init__(self): + # this maps file descriptors to keys + self._fd_to_key = {} + # read-only mapping returned by get_map() + self._map = _SelectorMapping(self) + + def _fileobj_lookup(self, fileobj): + """Return a file descriptor from a file object. + + This wraps _fileobj_to_fd() to do an exhaustive search in case + the object is invalid but we still have it in our map. This + is used by unregister() so we can unregister an object that + was previously registered even if it is closed. It is also + used by _SelectorMapping. + """ + try: + return _fileobj_to_fd(fileobj) + except ValueError: + # Do an exhaustive search. + for key in self._fd_to_key.values(): + if key.fileobj is fileobj: + return key.fd + # Raise ValueError after all. + raise + + def register(self, fileobj, events, data=None): + if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): + raise ValueError("Invalid events: {!r}".format(events)) + + key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) + + if key.fd in self._fd_to_key: + raise KeyError("{!r} (FD {}) is already registered" + .format(fileobj, key.fd)) + + self._fd_to_key[key.fd] = key + return key + + def unregister(self, fileobj): + try: + key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + return key + + def modify(self, fileobj, events, data=None): + # TODO: Subclasses can probably optimize this even further. + try: + key = self._fd_to_key[self._fileobj_lookup(fileobj)] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + if events != key.events: + self.unregister(fileobj) + key = self.register(fileobj, events, data) + elif data != key.data: + # Use a shortcut to update the data. + key = key._replace(data=data) + self._fd_to_key[key.fd] = key + return key + + def close(self): + self._fd_to_key.clear() + self._map = None + + def get_map(self): + return self._map + + def _key_from_fd(self, fd): + """Return the key associated to a given file descriptor. + + Parameters: + fd -- file descriptor + + Returns: + corresponding key, or None if not found + """ + try: + return self._fd_to_key[fd] + except KeyError: + return None + + +class SelectSelector(_BaseSelectorImpl): + """Select-based selector.""" + + def __init__(self): + super().__init__() + self._readers = set() + self._writers = set() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + if events & EVENT_READ: + self._readers.add(key.fd) + if events & EVENT_WRITE: + self._writers.add(key.fd) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._readers.discard(key.fd) + self._writers.discard(key.fd) + return key + + if sys.platform == 'win32': + def _select(self, r, w, _, timeout=None): + r, w, x = select.select(r, w, w, timeout) + return r, w + x, [] + else: + _select = select.select + + def select(self, timeout=None): + timeout = None if timeout is None else max(timeout, 0) + ready = [] + try: + r, w, _ = self._select(self._readers, self._writers, [], timeout) + except InterruptedError: + return ready + r = set(r) + w = set(w) + for fd in r | w: + events = 0 + if fd in r: + events |= EVENT_READ + if fd in w: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, 'poll'): + + class PollSelector(_BaseSelectorImpl): + """Poll-based selector.""" + + def __init__(self): + super().__init__() + self._poll = select.poll() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + poll_events = 0 + if events & EVENT_READ: + poll_events |= select.POLLIN + if events & EVENT_WRITE: + poll_events |= select.POLLOUT + self._poll.register(key.fd, poll_events) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._poll.unregister(key.fd) + return key + + def select(self, timeout=None): + if timeout is None: + timeout = None + elif timeout <= 0: + timeout = 0 + else: + # poll() has a resolution of 1 millisecond, round away from + # zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + ready = [] + try: + fd_event_list = self._poll.poll(timeout) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~select.POLLIN: + events |= EVENT_WRITE + if event & ~select.POLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, 'epoll'): + + class EpollSelector(_BaseSelectorImpl): + """Epoll-based selector.""" + + def __init__(self): + super().__init__() + self._epoll = select.epoll() + + def fileno(self): + return self._epoll.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + epoll_events = 0 + if events & EVENT_READ: + epoll_events |= select.EPOLLIN + if events & EVENT_WRITE: + epoll_events |= select.EPOLLOUT + try: + self._epoll.register(key.fd, epoll_events) + except BaseException: + super().unregister(fileobj) + raise + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + try: + self._epoll.unregister(key.fd) + except OSError: + # This can happen if the FD was closed since it + # was registered. + pass + return key + + def select(self, timeout=None): + if timeout is None: + timeout = -1 + elif timeout <= 0: + timeout = 0 + else: + # epoll_wait() has a resolution of 1 millisecond, round away + # from zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) * 1e-3 + + # epoll_wait() expects `maxevents` to be greater than zero; + # we want to make sure that `select()` can be called when no + # FD is registered. + max_ev = max(len(self._fd_to_key), 1) + + ready = [] + try: + fd_event_list = self._epoll.poll(timeout, max_ev) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~select.EPOLLIN: + events |= EVENT_WRITE + if event & ~select.EPOLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._epoll.close() + super().close() + + +if hasattr(select, 'devpoll'): + + class DevpollSelector(_BaseSelectorImpl): + """Solaris /dev/poll selector.""" + + def __init__(self): + super().__init__() + self._devpoll = select.devpoll() + + def fileno(self): + return self._devpoll.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + poll_events = 0 + if events & EVENT_READ: + poll_events |= select.POLLIN + if events & EVENT_WRITE: + poll_events |= select.POLLOUT + self._devpoll.register(key.fd, poll_events) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._devpoll.unregister(key.fd) + return key + + def select(self, timeout=None): + if timeout is None: + timeout = None + elif timeout <= 0: + timeout = 0 + else: + # devpoll() has a resolution of 1 millisecond, round away from + # zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + ready = [] + try: + fd_event_list = self._devpoll.poll(timeout) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~select.POLLIN: + events |= EVENT_WRITE + if event & ~select.POLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._devpoll.close() + super().close() + + +if hasattr(select, 'kqueue'): + + class KqueueSelector(_BaseSelectorImpl): + """Kqueue-based selector.""" + + def __init__(self): + super().__init__() + self._kqueue = select.kqueue() + + def fileno(self): + return self._kqueue.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + try: + if events & EVENT_READ: + kev = select.kevent(key.fd, select.KQ_FILTER_READ, + select.KQ_EV_ADD) + self._kqueue.control([kev], 0, 0) + if events & EVENT_WRITE: + kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, + select.KQ_EV_ADD) + self._kqueue.control([kev], 0, 0) + except BaseException: + super().unregister(fileobj) + raise + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + if key.events & EVENT_READ: + kev = select.kevent(key.fd, select.KQ_FILTER_READ, + select.KQ_EV_DELETE) + try: + self._kqueue.control([kev], 0, 0) + except OSError: + # This can happen if the FD was closed since it + # was registered. + pass + if key.events & EVENT_WRITE: + kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, + select.KQ_EV_DELETE) + try: + self._kqueue.control([kev], 0, 0) + except OSError: + # See comment above. + pass + return key + + def select(self, timeout=None): + timeout = None if timeout is None else max(timeout, 0) + max_ev = len(self._fd_to_key) + ready = [] + try: + kev_list = self._kqueue.control(None, max_ev, timeout) + except InterruptedError: + return ready + for kev in kev_list: + fd = kev.ident + flag = kev.filter + events = 0 + if flag == select.KQ_FILTER_READ: + events |= EVENT_READ + if flag == select.KQ_FILTER_WRITE: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._kqueue.close() + super().close() + + +# Choose the best implementation, roughly: +# epoll|kqueue|devpoll > poll > select. +# select() also can't accept a FD > FD_SETSIZE (usually around 1024) +if 'KqueueSelector' in globals(): + DefaultSelector = KqueueSelector +elif 'EpollSelector' in globals(): + DefaultSelector = EpollSelector +elif 'DevpollSelector' in globals(): + DefaultSelector = DevpollSelector +elif 'PollSelector' in globals(): + DefaultSelector = PollSelector +else: + DefaultSelector = SelectSelector diff --git a/thirdparty/asyncio/asyncio/sslproto.py b/thirdparty/asyncio/asyncio/sslproto.py new file mode 100644 index 0000000..804c5c3 --- /dev/null +++ b/thirdparty/asyncio/asyncio/sslproto.py @@ -0,0 +1,690 @@ +import collections +import warnings +try: + import ssl +except ImportError: # pragma: no cover + ssl = None + +from . import base_events +from . import compat +from . import protocols +from . import transports +from .log import logger + + +def _create_transport_context(server_side, server_hostname): + if server_side: + raise ValueError('Server side SSL needs a valid SSLContext') + + # Client side may pass ssl=True to use a default + # context; in that case the sslcontext passed is None. + # The default is secure for client connections. + if hasattr(ssl, 'create_default_context'): + # Python 3.4+: use up-to-date strong settings. + sslcontext = ssl.create_default_context() + if not server_hostname: + sslcontext.check_hostname = False + else: + # Fallback for Python 3.3. + sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.options |= ssl.OP_NO_SSLv3 + sslcontext.set_default_verify_paths() + sslcontext.verify_mode = ssl.CERT_REQUIRED + return sslcontext + + +def _is_sslproto_available(): + return hasattr(ssl, "MemoryBIO") + + +# States of an _SSLPipe. +_UNWRAPPED = "UNWRAPPED" +_DO_HANDSHAKE = "DO_HANDSHAKE" +_WRAPPED = "WRAPPED" +_SHUTDOWN = "SHUTDOWN" + + +class _SSLPipe(object): + """An SSL "Pipe". + + An SSL pipe allows you to communicate with an SSL/TLS protocol instance + through memory buffers. It can be used to implement a security layer for an + existing connection where you don't have access to the connection's file + descriptor, or for some reason you don't want to use it. + + An SSL pipe can be in "wrapped" and "unwrapped" mode. In unwrapped mode, + data is passed through untransformed. In wrapped mode, application level + data is encrypted to SSL record level data and vice versa. The SSL record + level is the lowest level in the SSL protocol suite and is what travels + as-is over the wire. + + An SslPipe initially is in "unwrapped" mode. To start SSL, call + do_handshake(). To shutdown SSL again, call unwrap(). + """ + + max_size = 256 * 1024 # Buffer size passed to read() + + def __init__(self, context, server_side, server_hostname=None): + """ + The *context* argument specifies the ssl.SSLContext to use. + + The *server_side* argument indicates whether this is a server side or + client side transport. + + The optional *server_hostname* argument can be used to specify the + hostname you are connecting to. You may only specify this parameter if + the _ssl module supports Server Name Indication (SNI). + """ + self._context = context + self._server_side = server_side + self._server_hostname = server_hostname + self._state = _UNWRAPPED + self._incoming = ssl.MemoryBIO() + self._outgoing = ssl.MemoryBIO() + self._sslobj = None + self._need_ssldata = False + self._handshake_cb = None + self._shutdown_cb = None + + @property + def context(self): + """The SSL context passed to the constructor.""" + return self._context + + @property + def ssl_object(self): + """The internal ssl.SSLObject instance. + + Return None if the pipe is not wrapped. + """ + return self._sslobj + + @property + def need_ssldata(self): + """Whether more record level data is needed to complete a handshake + that is currently in progress.""" + return self._need_ssldata + + @property + def wrapped(self): + """ + Whether a security layer is currently in effect. + + Return False during handshake. + """ + return self._state == _WRAPPED + + def do_handshake(self, callback=None): + """Start the SSL handshake. + + Return a list of ssldata. A ssldata element is a list of buffers + + The optional *callback* argument can be used to install a callback that + will be called when the handshake is complete. The callback will be + called with None if successful, else an exception instance. + """ + if self._state != _UNWRAPPED: + raise RuntimeError('handshake in progress or completed') + self._sslobj = self._context.wrap_bio( + self._incoming, self._outgoing, + server_side=self._server_side, + server_hostname=self._server_hostname) + self._state = _DO_HANDSHAKE + self._handshake_cb = callback + ssldata, appdata = self.feed_ssldata(b'', only_handshake=True) + assert len(appdata) == 0 + return ssldata + + def shutdown(self, callback=None): + """Start the SSL shutdown sequence. + + Return a list of ssldata. A ssldata element is a list of buffers + + The optional *callback* argument can be used to install a callback that + will be called when the shutdown is complete. The callback will be + called without arguments. + """ + if self._state == _UNWRAPPED: + raise RuntimeError('no security layer present') + if self._state == _SHUTDOWN: + raise RuntimeError('shutdown in progress') + assert self._state in (_WRAPPED, _DO_HANDSHAKE) + self._state = _SHUTDOWN + self._shutdown_cb = callback + ssldata, appdata = self.feed_ssldata(b'') + assert appdata == [] or appdata == [b''] + return ssldata + + def feed_eof(self): + """Send a potentially "ragged" EOF. + + This method will raise an SSL_ERROR_EOF exception if the EOF is + unexpected. + """ + self._incoming.write_eof() + ssldata, appdata = self.feed_ssldata(b'') + assert appdata == [] or appdata == [b''] + + def feed_ssldata(self, data, only_handshake=False): + """Feed SSL record level data into the pipe. + + The data must be a bytes instance. It is OK to send an empty bytes + instance. This can be used to get ssldata for a handshake initiated by + this endpoint. + + Return a (ssldata, appdata) tuple. The ssldata element is a list of + buffers containing SSL data that needs to be sent to the remote SSL. + + The appdata element is a list of buffers containing plaintext data that + needs to be forwarded to the application. The appdata list may contain + an empty buffer indicating an SSL "close_notify" alert. This alert must + be acknowledged by calling shutdown(). + """ + if self._state == _UNWRAPPED: + # If unwrapped, pass plaintext data straight through. + if data: + appdata = [data] + else: + appdata = [] + return ([], appdata) + + self._need_ssldata = False + if data: + self._incoming.write(data) + + ssldata = [] + appdata = [] + try: + if self._state == _DO_HANDSHAKE: + # Call do_handshake() until it doesn't raise anymore. + self._sslobj.do_handshake() + self._state = _WRAPPED + if self._handshake_cb: + self._handshake_cb(None) + if only_handshake: + return (ssldata, appdata) + # Handshake done: execute the wrapped block + + if self._state == _WRAPPED: + # Main state: read data from SSL until close_notify + while True: + chunk = self._sslobj.read(self.max_size) + appdata.append(chunk) + if not chunk: # close_notify + break + + elif self._state == _SHUTDOWN: + # Call shutdown() until it doesn't raise anymore. + self._sslobj.unwrap() + self._sslobj = None + self._state = _UNWRAPPED + if self._shutdown_cb: + self._shutdown_cb() + + elif self._state == _UNWRAPPED: + # Drain possible plaintext data after close_notify. + appdata.append(self._incoming.read()) + except (ssl.SSLError, ssl.CertificateError) as exc: + if getattr(exc, 'errno', None) not in ( + ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE, + ssl.SSL_ERROR_SYSCALL): + if self._state == _DO_HANDSHAKE and self._handshake_cb: + self._handshake_cb(exc) + raise + self._need_ssldata = (exc.errno == ssl.SSL_ERROR_WANT_READ) + + # Check for record level data that needs to be sent back. + # Happens for the initial handshake and renegotiations. + if self._outgoing.pending: + ssldata.append(self._outgoing.read()) + return (ssldata, appdata) + + def feed_appdata(self, data, offset=0): + """Feed plaintext data into the pipe. + + Return an (ssldata, offset) tuple. The ssldata element is a list of + buffers containing record level data that needs to be sent to the + remote SSL instance. The offset is the number of plaintext bytes that + were processed, which may be less than the length of data. + + NOTE: In case of short writes, this call MUST be retried with the SAME + buffer passed into the *data* argument (i.e. the id() must be the + same). This is an OpenSSL requirement. A further particularity is that + a short write will always have offset == 0, because the _ssl module + does not enable partial writes. And even though the offset is zero, + there will still be encrypted data in ssldata. + """ + assert 0 <= offset <= len(data) + if self._state == _UNWRAPPED: + # pass through data in unwrapped mode + if offset < len(data): + ssldata = [data[offset:]] + else: + ssldata = [] + return (ssldata, len(data)) + + ssldata = [] + view = memoryview(data) + while True: + self._need_ssldata = False + try: + if offset < len(view): + offset += self._sslobj.write(view[offset:]) + except ssl.SSLError as exc: + # It is not allowed to call write() after unwrap() until the + # close_notify is acknowledged. We return the condition to the + # caller as a short write. + if exc.reason == 'PROTOCOL_IS_SHUTDOWN': + exc.errno = ssl.SSL_ERROR_WANT_READ + if exc.errno not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE, + ssl.SSL_ERROR_SYSCALL): + raise + self._need_ssldata = (exc.errno == ssl.SSL_ERROR_WANT_READ) + + # See if there's any record level data back for us. + if self._outgoing.pending: + ssldata.append(self._outgoing.read()) + if offset == len(view) or self._need_ssldata: + break + return (ssldata, offset) + + +class _SSLProtocolTransport(transports._FlowControlMixin, + transports.Transport): + + def __init__(self, loop, ssl_protocol, app_protocol): + self._loop = loop + # SSLProtocol instance + self._ssl_protocol = ssl_protocol + self._app_protocol = app_protocol + self._closed = False + + def get_extra_info(self, name, default=None): + """Get optional transport information.""" + return self._ssl_protocol._get_extra_info(name, default) + + def set_protocol(self, protocol): + self._app_protocol = protocol + + def get_protocol(self): + return self._app_protocol + + def is_closing(self): + return self._closed + + def close(self): + """Close the transport. + + Buffered data will be flushed asynchronously. No more data + will be received. After all buffered data is flushed, the + protocol's connection_lost() method will (eventually) called + with None as its argument. + """ + self._closed = True + self._ssl_protocol._start_shutdown() + + # On Python 3.3 and older, objects with a destructor part of a reference + # cycle are never destroyed. It's not more the case on Python 3.4 thanks + # to the PEP 442. + if compat.PY34: + def __del__(self): + if not self._closed: + warnings.warn("unclosed transport %r" % self, ResourceWarning) + self.close() + + def pause_reading(self): + """Pause the receiving end. + + No data will be passed to the protocol's data_received() + method until resume_reading() is called. + """ + self._ssl_protocol._transport.pause_reading() + + def resume_reading(self): + """Resume the receiving end. + + Data received will once again be passed to the protocol's + data_received() method. + """ + self._ssl_protocol._transport.resume_reading() + + def set_write_buffer_limits(self, high=None, low=None): + """Set the high- and low-water limits for write flow control. + + These two values control when to call the protocol's + pause_writing() and resume_writing() methods. If specified, + the low-water limit must be less than or equal to the + high-water limit. Neither value can be negative. + + The defaults are implementation-specific. If only the + high-water limit is given, the low-water limit defaults to an + implementation-specific value less than or equal to the + high-water limit. Setting high to zero forces low to zero as + well, and causes pause_writing() to be called whenever the + buffer becomes non-empty. Setting low to zero causes + resume_writing() to be called only once the buffer is empty. + Use of zero for either limit is generally sub-optimal as it + reduces opportunities for doing I/O and computation + concurrently. + """ + self._ssl_protocol._transport.set_write_buffer_limits(high, low) + + def get_write_buffer_size(self): + """Return the current size of the write buffer.""" + return self._ssl_protocol._transport.get_write_buffer_size() + + def write(self, data): + """Write some data bytes to the transport. + + This does not block; it buffers the data and arranges for it + to be sent out asynchronously. + """ + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("data: expecting a bytes-like instance, got {!r}" + .format(type(data).__name__)) + if not data: + return + self._ssl_protocol._write_appdata(data) + + def can_write_eof(self): + """Return True if this transport supports write_eof(), False if not.""" + return False + + def abort(self): + """Close the transport immediately. + + Buffered data will be lost. No more data will be received. + The protocol's connection_lost() method will (eventually) be + called with None as its argument. + """ + self._ssl_protocol._abort() + + +class SSLProtocol(protocols.Protocol): + """SSL protocol. + + Implementation of SSL on top of a socket using incoming and outgoing + buffers which are ssl.MemoryBIO objects. + """ + + def __init__(self, loop, app_protocol, sslcontext, waiter, + server_side=False, server_hostname=None, + call_connection_made=True): + if ssl is None: + raise RuntimeError('stdlib ssl module not available') + + if not sslcontext: + sslcontext = _create_transport_context(server_side, server_hostname) + + self._server_side = server_side + if server_hostname and not server_side: + self._server_hostname = server_hostname + else: + self._server_hostname = None + self._sslcontext = sslcontext + # SSL-specific extra info. More info are set when the handshake + # completes. + self._extra = dict(sslcontext=sslcontext) + + # App data write buffering + self._write_backlog = collections.deque() + self._write_buffer_size = 0 + + self._waiter = waiter + self._loop = loop + self._app_protocol = app_protocol + self._app_transport = _SSLProtocolTransport(self._loop, + self, self._app_protocol) + # _SSLPipe instance (None until the connection is made) + self._sslpipe = None + self._session_established = False + self._in_handshake = False + self._in_shutdown = False + # transport, ex: SelectorSocketTransport + self._transport = None + self._call_connection_made = call_connection_made + + def _wakeup_waiter(self, exc=None): + if self._waiter is None: + return + if not self._waiter.cancelled(): + if exc is not None: + self._waiter.set_exception(exc) + else: + self._waiter.set_result(None) + self._waiter = None + + def connection_made(self, transport): + """Called when the low-level connection is made. + + Start the SSL handshake. + """ + self._transport = transport + self._sslpipe = _SSLPipe(self._sslcontext, + self._server_side, + self._server_hostname) + self._start_handshake() + + def connection_lost(self, exc): + """Called when the low-level connection is lost or closed. + + The argument is an exception object or None (the latter + meaning a regular EOF is received or the connection was + aborted or closed). + """ + if self._session_established: + self._session_established = False + self._loop.call_soon(self._app_protocol.connection_lost, exc) + self._transport = None + self._app_transport = None + + def pause_writing(self): + """Called when the low-level transport's buffer goes over + the high-water mark. + """ + self._app_protocol.pause_writing() + + def resume_writing(self): + """Called when the low-level transport's buffer drains below + the low-water mark. + """ + self._app_protocol.resume_writing() + + def data_received(self, data): + """Called when some SSL data is received. + + The argument is a bytes object. + """ + try: + ssldata, appdata = self._sslpipe.feed_ssldata(data) + except ssl.SSLError as e: + if self._loop.get_debug(): + logger.warning('%r: SSL error %s (reason %s)', + self, e.errno, e.reason) + self._abort() + return + + for chunk in ssldata: + self._transport.write(chunk) + + for chunk in appdata: + if chunk: + self._app_protocol.data_received(chunk) + else: + self._start_shutdown() + break + + def eof_received(self): + """Called when the other end of the low-level stream + is half-closed. + + If this returns a false value (including None), the transport + will close itself. If it returns a true value, closing the + transport is up to the protocol. + """ + try: + if self._loop.get_debug(): + logger.debug("%r received EOF", self) + + self._wakeup_waiter(ConnectionResetError) + + if not self._in_handshake: + keep_open = self._app_protocol.eof_received() + if keep_open: + logger.warning('returning true from eof_received() ' + 'has no effect when using ssl') + finally: + self._transport.close() + + def _get_extra_info(self, name, default=None): + if name in self._extra: + return self._extra[name] + else: + return self._transport.get_extra_info(name, default) + + def _start_shutdown(self): + if self._in_shutdown: + return + self._in_shutdown = True + self._write_appdata(b'') + + def _write_appdata(self, data): + self._write_backlog.append((data, 0)) + self._write_buffer_size += len(data) + self._process_write_backlog() + + def _start_handshake(self): + if self._loop.get_debug(): + logger.debug("%r starts SSL handshake", self) + self._handshake_start_time = self._loop.time() + else: + self._handshake_start_time = None + self._in_handshake = True + # (b'', 1) is a special value in _process_write_backlog() to do + # the SSL handshake + self._write_backlog.append((b'', 1)) + self._loop.call_soon(self._process_write_backlog) + + def _on_handshake_complete(self, handshake_exc): + self._in_handshake = False + + sslobj = self._sslpipe.ssl_object + try: + if handshake_exc is not None: + raise handshake_exc + + peercert = sslobj.getpeercert() + if not hasattr(self._sslcontext, 'check_hostname'): + # Verify hostname if requested, Python 3.4+ uses check_hostname + # and checks the hostname in do_handshake() + if (self._server_hostname + and self._sslcontext.verify_mode != ssl.CERT_NONE): + ssl.match_hostname(peercert, self._server_hostname) + except BaseException as exc: + if self._loop.get_debug(): + if isinstance(exc, ssl.CertificateError): + logger.warning("%r: SSL handshake failed " + "on verifying the certificate", + self, exc_info=True) + else: + logger.warning("%r: SSL handshake failed", + self, exc_info=True) + self._transport.close() + if isinstance(exc, Exception): + self._wakeup_waiter(exc) + return + else: + raise + + if self._loop.get_debug(): + dt = self._loop.time() - self._handshake_start_time + logger.debug("%r: SSL handshake took %.1f ms", self, dt * 1e3) + + # Add extra info that becomes available after handshake. + self._extra.update(peercert=peercert, + cipher=sslobj.cipher(), + compression=sslobj.compression(), + ssl_object=sslobj, + ) + if self._call_connection_made: + self._app_protocol.connection_made(self._app_transport) + self._wakeup_waiter() + self._session_established = True + # In case transport.write() was already called. Don't call + # immediately _process_write_backlog(), but schedule it: + # _on_handshake_complete() can be called indirectly from + # _process_write_backlog(), and _process_write_backlog() is not + # reentrant. + self._loop.call_soon(self._process_write_backlog) + + def _process_write_backlog(self): + # Try to make progress on the write backlog. + if self._transport is None: + return + + try: + for i in range(len(self._write_backlog)): + data, offset = self._write_backlog[0] + if data: + ssldata, offset = self._sslpipe.feed_appdata(data, offset) + elif offset: + ssldata = self._sslpipe.do_handshake( + self._on_handshake_complete) + offset = 1 + else: + ssldata = self._sslpipe.shutdown(self._finalize) + offset = 1 + + for chunk in ssldata: + self._transport.write(chunk) + + if offset < len(data): + self._write_backlog[0] = (data, offset) + # A short write means that a write is blocked on a read + # We need to enable reading if it is paused! + assert self._sslpipe.need_ssldata + if self._transport._paused: + self._transport.resume_reading() + break + + # An entire chunk from the backlog was processed. We can + # delete it and reduce the outstanding buffer size. + del self._write_backlog[0] + self._write_buffer_size -= len(data) + except BaseException as exc: + if self._in_handshake: + # BaseExceptions will be re-raised in _on_handshake_complete. + self._on_handshake_complete(exc) + else: + self._fatal_error(exc, 'Fatal error on SSL transport') + if not isinstance(exc, Exception): + # BaseException + raise + + def _fatal_error(self, exc, message='Fatal error on transport'): + # Should be called from exception handler only. + if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if self._loop.get_debug(): + logger.debug("%r: %s", self, message, exc_info=True) + else: + self._loop.call_exception_handler({ + 'message': message, + 'exception': exc, + 'transport': self._transport, + 'protocol': self, + }) + if self._transport: + self._transport._force_close(exc) + + def _finalize(self): + if self._transport is not None: + self._transport.close() + + def _abort(self): + if self._transport is not None: + try: + self._transport.abort() + finally: + self._finalize() diff --git a/thirdparty/asyncio/asyncio/streams.py b/thirdparty/asyncio/asyncio/streams.py new file mode 100644 index 0000000..a82cc79 --- /dev/null +++ b/thirdparty/asyncio/asyncio/streams.py @@ -0,0 +1,695 @@ +"""Stream-related things.""" + +__all__ = ['StreamReader', 'StreamWriter', 'StreamReaderProtocol', + 'open_connection', 'start_server', + 'IncompleteReadError', + 'LimitOverrunError', + ] + +import socket + +if hasattr(socket, 'AF_UNIX'): + __all__.extend(['open_unix_connection', 'start_unix_server']) + +from . import coroutines +from . import compat +from . import events +from . import protocols +from .coroutines import coroutine +from .log import logger + + +_DEFAULT_LIMIT = 2 ** 16 + + +class IncompleteReadError(EOFError): + """ + Incomplete read error. Attributes: + + - partial: read bytes string before the end of stream was reached + - expected: total number of expected bytes (or None if unknown) + """ + def __init__(self, partial, expected): + super().__init__("%d bytes read on a total of %r expected bytes" + % (len(partial), expected)) + self.partial = partial + self.expected = expected + + +class LimitOverrunError(Exception): + """Reached the buffer limit while looking for a separator. + + Attributes: + - consumed: total number of to be consumed bytes. + """ + def __init__(self, message, consumed): + super().__init__(message) + self.consumed = consumed + + +@coroutine +def open_connection(host=None, port=None, *, + loop=None, limit=_DEFAULT_LIMIT, **kwds): + """A wrapper for create_connection() returning a (reader, writer) pair. + + The reader returned is a StreamReader instance; the writer is a + StreamWriter instance. + + The arguments are all the usual arguments to create_connection() + except protocol_factory; most common are positional host and port, + with various optional keyword arguments following. + + Additional optional keyword arguments are loop (to set the event loop + instance to use) and limit (to set the buffer limit passed to the + StreamReader). + + (If you want to customize the StreamReader and/or + StreamReaderProtocol classes, just copy the code -- there's + really nothing special here except some convenience.) + """ + if loop is None: + loop = events.get_event_loop() + reader = StreamReader(limit=limit, loop=loop) + protocol = StreamReaderProtocol(reader, loop=loop) + transport, _ = yield from loop.create_connection( + lambda: protocol, host, port, **kwds) + writer = StreamWriter(transport, protocol, reader, loop) + return reader, writer + + +@coroutine +def start_server(client_connected_cb, host=None, port=None, *, + loop=None, limit=_DEFAULT_LIMIT, **kwds): + """Start a socket server, call back for each client connected. + + The first parameter, `client_connected_cb`, takes two parameters: + client_reader, client_writer. client_reader is a StreamReader + object, while client_writer is a StreamWriter object. This + parameter can either be a plain callback function or a coroutine; + if it is a coroutine, it will be automatically converted into a + Task. + + The rest of the arguments are all the usual arguments to + loop.create_server() except protocol_factory; most common are + positional host and port, with various optional keyword arguments + following. The return value is the same as loop.create_server(). + + Additional optional keyword arguments are loop (to set the event loop + instance to use) and limit (to set the buffer limit passed to the + StreamReader). + + The return value is the same as loop.create_server(), i.e. a + Server object which can be used to stop the service. + """ + if loop is None: + loop = events.get_event_loop() + + def factory(): + reader = StreamReader(limit=limit, loop=loop) + protocol = StreamReaderProtocol(reader, client_connected_cb, + loop=loop) + return protocol + + return (yield from loop.create_server(factory, host, port, **kwds)) + + +if hasattr(socket, 'AF_UNIX'): + # UNIX Domain Sockets are supported on this platform + + @coroutine + def open_unix_connection(path=None, *, + loop=None, limit=_DEFAULT_LIMIT, **kwds): + """Similar to `open_connection` but works with UNIX Domain Sockets.""" + if loop is None: + loop = events.get_event_loop() + reader = StreamReader(limit=limit, loop=loop) + protocol = StreamReaderProtocol(reader, loop=loop) + transport, _ = yield from loop.create_unix_connection( + lambda: protocol, path, **kwds) + writer = StreamWriter(transport, protocol, reader, loop) + return reader, writer + + @coroutine + def start_unix_server(client_connected_cb, path=None, *, + loop=None, limit=_DEFAULT_LIMIT, **kwds): + """Similar to `start_server` but works with UNIX Domain Sockets.""" + if loop is None: + loop = events.get_event_loop() + + def factory(): + reader = StreamReader(limit=limit, loop=loop) + protocol = StreamReaderProtocol(reader, client_connected_cb, + loop=loop) + return protocol + + return (yield from loop.create_unix_server(factory, path, **kwds)) + + +class FlowControlMixin(protocols.Protocol): + """Reusable flow control logic for StreamWriter.drain(). + + This implements the protocol methods pause_writing(), + resume_reading() and connection_lost(). If the subclass overrides + these it must call the super methods. + + StreamWriter.drain() must wait for _drain_helper() coroutine. + """ + + def __init__(self, loop=None): + if loop is None: + self._loop = events.get_event_loop() + else: + self._loop = loop + self._paused = False + self._drain_waiter = None + self._connection_lost = False + + def pause_writing(self): + assert not self._paused + self._paused = True + if self._loop.get_debug(): + logger.debug("%r pauses writing", self) + + def resume_writing(self): + assert self._paused + self._paused = False + if self._loop.get_debug(): + logger.debug("%r resumes writing", self) + + waiter = self._drain_waiter + if waiter is not None: + self._drain_waiter = None + if not waiter.done(): + waiter.set_result(None) + + def connection_lost(self, exc): + self._connection_lost = True + # Wake up the writer if currently paused. + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + return + self._drain_waiter = None + if waiter.done(): + return + if exc is None: + waiter.set_result(None) + else: + waiter.set_exception(exc) + + @coroutine + def _drain_helper(self): + if self._connection_lost: + raise ConnectionResetError('Connection lost') + if not self._paused: + return + waiter = self._drain_waiter + assert waiter is None or waiter.cancelled() + waiter = self._loop.create_future() + self._drain_waiter = waiter + yield from waiter + + +class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): + """Helper class to adapt between Protocol and StreamReader. + + (This is a helper class instead of making StreamReader itself a + Protocol subclass, because the StreamReader has other potential + uses, and to prevent the user of the StreamReader to accidentally + call inappropriate methods of the protocol.) + """ + + def __init__(self, stream_reader, client_connected_cb=None, loop=None): + super().__init__(loop=loop) + self._stream_reader = stream_reader + self._stream_writer = None + self._client_connected_cb = client_connected_cb + self._over_ssl = False + + def connection_made(self, transport): + self._stream_reader.set_transport(transport) + self._over_ssl = transport.get_extra_info('sslcontext') is not None + if self._client_connected_cb is not None: + self._stream_writer = StreamWriter(transport, self, + self._stream_reader, + self._loop) + res = self._client_connected_cb(self._stream_reader, + self._stream_writer) + if coroutines.iscoroutine(res): + self._loop.create_task(res) + + def connection_lost(self, exc): + if self._stream_reader is not None: + if exc is None: + self._stream_reader.feed_eof() + else: + self._stream_reader.set_exception(exc) + super().connection_lost(exc) + self._stream_reader = None + self._stream_writer = None + + def data_received(self, data): + self._stream_reader.feed_data(data) + + def eof_received(self): + self._stream_reader.feed_eof() + if self._over_ssl: + # Prevent a warning in SSLProtocol.eof_received: + # "returning true from eof_received() + # has no effect when using ssl" + return False + return True + + +class StreamWriter: + """Wraps a Transport. + + This exposes write(), writelines(), [can_]write_eof(), + get_extra_info() and close(). It adds drain() which returns an + optional Future on which you can wait for flow control. It also + adds a transport property which references the Transport + directly. + """ + + def __init__(self, transport, protocol, reader, loop): + self._transport = transport + self._protocol = protocol + # drain() expects that the reader has an exception() method + assert reader is None or isinstance(reader, StreamReader) + self._reader = reader + self._loop = loop + + def __repr__(self): + info = [self.__class__.__name__, 'transport=%r' % self._transport] + if self._reader is not None: + info.append('reader=%r' % self._reader) + return '<%s>' % ' '.join(info) + + @property + def transport(self): + return self._transport + + def write(self, data): + self._transport.write(data) + + def writelines(self, data): + self._transport.writelines(data) + + def write_eof(self): + return self._transport.write_eof() + + def can_write_eof(self): + return self._transport.can_write_eof() + + def close(self): + return self._transport.close() + + def get_extra_info(self, name, default=None): + return self._transport.get_extra_info(name, default) + + @coroutine + def drain(self): + """Flush the write buffer. + + The intended use is to write + + w.write(data) + yield from w.drain() + """ + if self._reader is not None: + exc = self._reader.exception() + if exc is not None: + raise exc + if self._transport is not None: + if self._transport.is_closing(): + # Yield to the event loop so connection_lost() may be + # called. Without this, _drain_helper() would return + # immediately, and code that calls + # write(...); yield from drain() + # in a loop would never call connection_lost(), so it + # would not see an error when the socket is closed. + yield + yield from self._protocol._drain_helper() + + +class StreamReader: + + def __init__(self, limit=_DEFAULT_LIMIT, loop=None): + # The line length limit is a security feature; + # it also doubles as half the buffer limit. + + if limit <= 0: + raise ValueError('Limit cannot be <= 0') + + self._limit = limit + if loop is None: + self._loop = events.get_event_loop() + else: + self._loop = loop + self._buffer = bytearray() + self._eof = False # Whether we're done. + self._waiter = None # A future used by _wait_for_data() + self._exception = None + self._transport = None + self._paused = False + + def __repr__(self): + info = ['StreamReader'] + if self._buffer: + info.append('%d bytes' % len(self._buffer)) + if self._eof: + info.append('eof') + if self._limit != _DEFAULT_LIMIT: + info.append('l=%d' % self._limit) + if self._waiter: + info.append('w=%r' % self._waiter) + if self._exception: + info.append('e=%r' % self._exception) + if self._transport: + info.append('t=%r' % self._transport) + if self._paused: + info.append('paused') + return '<%s>' % ' '.join(info) + + def exception(self): + return self._exception + + def set_exception(self, exc): + self._exception = exc + + waiter = self._waiter + if waiter is not None: + self._waiter = None + if not waiter.cancelled(): + waiter.set_exception(exc) + + def _wakeup_waiter(self): + """Wakeup read*() functions waiting for data or EOF.""" + waiter = self._waiter + if waiter is not None: + self._waiter = None + if not waiter.cancelled(): + waiter.set_result(None) + + def set_transport(self, transport): + assert self._transport is None, 'Transport already set' + self._transport = transport + + def _maybe_resume_transport(self): + if self._paused and len(self._buffer) <= self._limit: + self._paused = False + self._transport.resume_reading() + + def feed_eof(self): + self._eof = True + self._wakeup_waiter() + + def at_eof(self): + """Return True if the buffer is empty and 'feed_eof' was called.""" + return self._eof and not self._buffer + + def feed_data(self, data): + assert not self._eof, 'feed_data after feed_eof' + + if not data: + return + + self._buffer.extend(data) + self._wakeup_waiter() + + if (self._transport is not None and + not self._paused and + len(self._buffer) > 2 * self._limit): + try: + self._transport.pause_reading() + except NotImplementedError: + # The transport can't be paused. + # We'll just have to buffer all data. + # Forget the transport so we don't keep trying. + self._transport = None + else: + self._paused = True + + @coroutine + def _wait_for_data(self, func_name): + """Wait until feed_data() or feed_eof() is called. + + If stream was paused, automatically resume it. + """ + # StreamReader uses a future to link the protocol feed_data() method + # to a read coroutine. Running two read coroutines at the same time + # would have an unexpected behaviour. It would not possible to know + # which coroutine would get the next data. + if self._waiter is not None: + raise RuntimeError('%s() called while another coroutine is ' + 'already waiting for incoming data' % func_name) + + assert not self._eof, '_wait_for_data after EOF' + + # Waiting for data while paused will make deadlock, so prevent it. + # This is essential for readexactly(n) for case when n > self._limit. + if self._paused: + self._paused = False + self._transport.resume_reading() + + self._waiter = self._loop.create_future() + try: + yield from self._waiter + finally: + self._waiter = None + + @coroutine + def readline(self): + """Read chunk of data from the stream until newline (b'\n') is found. + + On success, return chunk that ends with newline. If only partial + line can be read due to EOF, return incomplete line without + terminating newline. When EOF was reached while no bytes read, empty + bytes object is returned. + + If limit is reached, ValueError will be raised. In that case, if + newline was found, complete line including newline will be removed + from internal buffer. Else, internal buffer will be cleared. Limit is + compared against part of the line without newline. + + If stream was paused, this function will automatically resume it if + needed. + """ + sep = b'\n' + seplen = len(sep) + try: + line = yield from self.readuntil(sep) + except IncompleteReadError as e: + return e.partial + except LimitOverrunError as e: + if self._buffer.startswith(sep, e.consumed): + del self._buffer[:e.consumed + seplen] + else: + self._buffer.clear() + self._maybe_resume_transport() + raise ValueError(e.args[0]) + return line + + @coroutine + def readuntil(self, separator=b'\n'): + """Read data from the stream until ``separator`` is found. + + On success, the data and separator will be removed from the + internal buffer (consumed). Returned data will include the + separator at the end. + + Configured stream limit is used to check result. Limit sets the + maximal length of data that can be returned, not counting the + separator. + + If an EOF occurs and the complete separator is still not found, + an IncompleteReadError exception will be raised, and the internal + buffer will be reset. The IncompleteReadError.partial attribute + may contain the separator partially. + + If the data cannot be read because of over limit, a + LimitOverrunError exception will be raised, and the data + will be left in the internal buffer, so it can be read again. + """ + seplen = len(separator) + if seplen == 0: + raise ValueError('Separator should be at least one-byte string') + + if self._exception is not None: + raise self._exception + + # Consume whole buffer except last bytes, which length is + # one less than seplen. Let's check corner cases with + # separator='SEPARATOR': + # * we have received almost complete separator (without last + # byte). i.e buffer='some textSEPARATO'. In this case we + # can safely consume len(separator) - 1 bytes. + # * last byte of buffer is first byte of separator, i.e. + # buffer='abcdefghijklmnopqrS'. We may safely consume + # everything except that last byte, but this require to + # analyze bytes of buffer that match partial separator. + # This is slow and/or require FSM. For this case our + # implementation is not optimal, since require rescanning + # of data that is known to not belong to separator. In + # real world, separator will not be so long to notice + # performance problems. Even when reading MIME-encoded + # messages :) + + # `offset` is the number of bytes from the beginning of the buffer + # where there is no occurrence of `separator`. + offset = 0 + + # Loop until we find `separator` in the buffer, exceed the buffer size, + # or an EOF has happened. + while True: + buflen = len(self._buffer) + + # Check if we now have enough data in the buffer for `separator` to + # fit. + if buflen - offset >= seplen: + isep = self._buffer.find(separator, offset) + + if isep != -1: + # `separator` is in the buffer. `isep` will be used later + # to retrieve the data. + break + + # see upper comment for explanation. + offset = buflen + 1 - seplen + if offset > self._limit: + raise LimitOverrunError( + 'Separator is not found, and chunk exceed the limit', + offset) + + # Complete message (with full separator) may be present in buffer + # even when EOF flag is set. This may happen when the last chunk + # adds data which makes separator be found. That's why we check for + # EOF *ater* inspecting the buffer. + if self._eof: + chunk = bytes(self._buffer) + self._buffer.clear() + raise IncompleteReadError(chunk, None) + + # _wait_for_data() will resume reading if stream was paused. + yield from self._wait_for_data('readuntil') + + if isep > self._limit: + raise LimitOverrunError( + 'Separator is found, but chunk is longer than limit', isep) + + chunk = self._buffer[:isep + seplen] + del self._buffer[:isep + seplen] + self._maybe_resume_transport() + return bytes(chunk) + + @coroutine + def read(self, n=-1): + """Read up to `n` bytes from the stream. + + If n is not provided, or set to -1, read until EOF and return all read + bytes. If the EOF was received and the internal buffer is empty, return + an empty bytes object. + + If n is zero, return empty bytes object immediately. + + If n is positive, this function try to read `n` bytes, and may return + less or equal bytes than requested, but at least one byte. If EOF was + received before any byte is read, this function returns empty byte + object. + + Returned value is not limited with limit, configured at stream + creation. + + If stream was paused, this function will automatically resume it if + needed. + """ + + if self._exception is not None: + raise self._exception + + if n == 0: + return b'' + + if n < 0: + # This used to just loop creating a new waiter hoping to + # collect everything in self._buffer, but that would + # deadlock if the subprocess sends more than self.limit + # bytes. So just call self.read(self._limit) until EOF. + blocks = [] + while True: + block = yield from self.read(self._limit) + if not block: + break + blocks.append(block) + return b''.join(blocks) + + if not self._buffer and not self._eof: + yield from self._wait_for_data('read') + + # This will work right even if buffer is less than n bytes + data = bytes(self._buffer[:n]) + del self._buffer[:n] + + self._maybe_resume_transport() + return data + + @coroutine + def readexactly(self, n): + """Read exactly `n` bytes. + + Raise an IncompleteReadError if EOF is reached before `n` bytes can be + read. The IncompleteReadError.partial attribute of the exception will + contain the partial read bytes. + + if n is zero, return empty bytes object. + + Returned value is not limited with limit, configured at stream + creation. + + If stream was paused, this function will automatically resume it if + needed. + """ + if n < 0: + raise ValueError('readexactly size can not be less than zero') + + if self._exception is not None: + raise self._exception + + if n == 0: + return b'' + + while len(self._buffer) < n: + if self._eof: + incomplete = bytes(self._buffer) + self._buffer.clear() + raise IncompleteReadError(incomplete, n) + + yield from self._wait_for_data('readexactly') + + if len(self._buffer) == n: + data = bytes(self._buffer) + self._buffer.clear() + else: + data = bytes(self._buffer[:n]) + del self._buffer[:n] + self._maybe_resume_transport() + return data + + if compat.PY35: + @coroutine + def __aiter__(self): + return self + + @coroutine + def __anext__(self): + val = yield from self.readline() + if val == b'': + raise StopAsyncIteration + return val + + if compat.PY352: + # In Python 3.5.2 and greater, __aiter__ should return + # the asynchronous iterator directly. + def __aiter__(self): + return self diff --git a/thirdparty/asyncio/asyncio/subprocess.py b/thirdparty/asyncio/asyncio/subprocess.py new file mode 100644 index 0000000..b2f5304 --- /dev/null +++ b/thirdparty/asyncio/asyncio/subprocess.py @@ -0,0 +1,213 @@ +__all__ = ['create_subprocess_exec', 'create_subprocess_shell'] + +import subprocess + +from . import events +from . import protocols +from . import streams +from . import tasks +from .coroutines import coroutine +from .log import logger + + +PIPE = subprocess.PIPE +STDOUT = subprocess.STDOUT +DEVNULL = subprocess.DEVNULL + + +class SubprocessStreamProtocol(streams.FlowControlMixin, + protocols.SubprocessProtocol): + """Like StreamReaderProtocol, but for a subprocess.""" + + def __init__(self, limit, loop): + super().__init__(loop=loop) + self._limit = limit + self.stdin = self.stdout = self.stderr = None + self._transport = None + + def __repr__(self): + info = [self.__class__.__name__] + if self.stdin is not None: + info.append('stdin=%r' % self.stdin) + if self.stdout is not None: + info.append('stdout=%r' % self.stdout) + if self.stderr is not None: + info.append('stderr=%r' % self.stderr) + return '<%s>' % ' '.join(info) + + def connection_made(self, transport): + self._transport = transport + + stdout_transport = transport.get_pipe_transport(1) + if stdout_transport is not None: + self.stdout = streams.StreamReader(limit=self._limit, + loop=self._loop) + self.stdout.set_transport(stdout_transport) + + stderr_transport = transport.get_pipe_transport(2) + if stderr_transport is not None: + self.stderr = streams.StreamReader(limit=self._limit, + loop=self._loop) + self.stderr.set_transport(stderr_transport) + + stdin_transport = transport.get_pipe_transport(0) + if stdin_transport is not None: + self.stdin = streams.StreamWriter(stdin_transport, + protocol=self, + reader=None, + loop=self._loop) + + def pipe_data_received(self, fd, data): + if fd == 1: + reader = self.stdout + elif fd == 2: + reader = self.stderr + else: + reader = None + if reader is not None: + reader.feed_data(data) + + def pipe_connection_lost(self, fd, exc): + if fd == 0: + pipe = self.stdin + if pipe is not None: + pipe.close() + self.connection_lost(exc) + return + if fd == 1: + reader = self.stdout + elif fd == 2: + reader = self.stderr + else: + reader = None + if reader != None: + if exc is None: + reader.feed_eof() + else: + reader.set_exception(exc) + + def process_exited(self): + self._transport.close() + self._transport = None + + +class Process: + def __init__(self, transport, protocol, loop): + self._transport = transport + self._protocol = protocol + self._loop = loop + self.stdin = protocol.stdin + self.stdout = protocol.stdout + self.stderr = protocol.stderr + self.pid = transport.get_pid() + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.pid) + + @property + def returncode(self): + return self._transport.get_returncode() + + @coroutine + def wait(self): + """Wait until the process exit and return the process return code. + + This method is a coroutine.""" + return (yield from self._transport._wait()) + + def send_signal(self, signal): + self._transport.send_signal(signal) + + def terminate(self): + self._transport.terminate() + + def kill(self): + self._transport.kill() + + @coroutine + def _feed_stdin(self, input): + debug = self._loop.get_debug() + self.stdin.write(input) + if debug: + logger.debug('%r communicate: feed stdin (%s bytes)', + self, len(input)) + try: + yield from self.stdin.drain() + except (BrokenPipeError, ConnectionResetError) as exc: + # communicate() ignores BrokenPipeError and ConnectionResetError + if debug: + logger.debug('%r communicate: stdin got %r', self, exc) + + if debug: + logger.debug('%r communicate: close stdin', self) + self.stdin.close() + + @coroutine + def _noop(self): + return None + + @coroutine + def _read_stream(self, fd): + transport = self._transport.get_pipe_transport(fd) + if fd == 2: + stream = self.stderr + else: + assert fd == 1 + stream = self.stdout + if self._loop.get_debug(): + name = 'stdout' if fd == 1 else 'stderr' + logger.debug('%r communicate: read %s', self, name) + output = yield from stream.read() + if self._loop.get_debug(): + name = 'stdout' if fd == 1 else 'stderr' + logger.debug('%r communicate: close %s', self, name) + transport.close() + return output + + @coroutine + def communicate(self, input=None): + if input is not None: + stdin = self._feed_stdin(input) + else: + stdin = self._noop() + if self.stdout is not None: + stdout = self._read_stream(1) + else: + stdout = self._noop() + if self.stderr is not None: + stderr = self._read_stream(2) + else: + stderr = self._noop() + stdin, stdout, stderr = yield from tasks.gather(stdin, stdout, stderr, + loop=self._loop) + yield from self.wait() + return (stdout, stderr) + + +@coroutine +def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, + loop=None, limit=streams._DEFAULT_LIMIT, **kwds): + if loop is None: + loop = events.get_event_loop() + protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, + loop=loop) + transport, protocol = yield from loop.subprocess_shell( + protocol_factory, + cmd, stdin=stdin, stdout=stdout, + stderr=stderr, **kwds) + return Process(transport, protocol, loop) + +@coroutine +def create_subprocess_exec(program, *args, stdin=None, stdout=None, + stderr=None, loop=None, + limit=streams._DEFAULT_LIMIT, **kwds): + if loop is None: + loop = events.get_event_loop() + protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, + loop=loop) + transport, protocol = yield from loop.subprocess_exec( + protocol_factory, + program, *args, + stdin=stdin, stdout=stdout, + stderr=stderr, **kwds) + return Process(transport, protocol, loop) diff --git a/thirdparty/asyncio/asyncio/tasks.py b/thirdparty/asyncio/asyncio/tasks.py new file mode 100644 index 0000000..8852aa5 --- /dev/null +++ b/thirdparty/asyncio/asyncio/tasks.py @@ -0,0 +1,757 @@ +"""Support for tasks, coroutines and the scheduler.""" + +__all__ = ['Task', + 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', + 'wait', 'wait_for', 'as_completed', 'sleep', 'async', + 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', + ] + +import concurrent.futures +import functools +import inspect +import linecache +import traceback +import warnings +import weakref + +from . import compat +from . import coroutines +from . import events +from . import futures +from .coroutines import coroutine + + +class Task(futures.Future): + """A coroutine wrapped in a Future.""" + + # An important invariant maintained while a Task not done: + # + # - Either _fut_waiter is None, and _step() is scheduled; + # - or _fut_waiter is some Future, and _step() is *not* scheduled. + # + # The only transition from the latter to the former is through + # _wakeup(). When _fut_waiter is not None, one of its callbacks + # must be _wakeup(). + + # Weak set containing all tasks alive. + _all_tasks = weakref.WeakSet() + + # Dictionary containing tasks that are currently active in + # all running event loops. {EventLoop: Task} + _current_tasks = {} + + # If False, don't log a message if the task is destroyed whereas its + # status is still pending + _log_destroy_pending = True + + @classmethod + def current_task(cls, loop=None): + """Return the currently running task in an event loop or None. + + By default the current task for the current event loop is returned. + + None is returned when called not in the context of a Task. + """ + if loop is None: + loop = events.get_event_loop() + return cls._current_tasks.get(loop) + + @classmethod + def all_tasks(cls, loop=None): + """Return a set of all tasks for an event loop. + + By default all tasks for the current event loop are returned. + """ + if loop is None: + loop = events.get_event_loop() + return {t for t in cls._all_tasks if t._loop is loop} + + def __init__(self, coro, *, loop=None): + assert coroutines.iscoroutine(coro), repr(coro) + super().__init__(loop=loop) + if self._source_traceback: + del self._source_traceback[-1] + self._coro = coro + self._fut_waiter = None + self._must_cancel = False + self._loop.call_soon(self._step) + self.__class__._all_tasks.add(self) + + # On Python 3.3 or older, objects with a destructor that are part of a + # reference cycle are never destroyed. That's not the case any more on + # Python 3.4 thanks to the PEP 442. + if compat.PY34: + def __del__(self): + if self._state == futures._PENDING and self._log_destroy_pending: + context = { + 'task': self, + 'message': 'Task was destroyed but it is pending!', + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + futures.Future.__del__(self) + + def _repr_info(self): + info = super()._repr_info() + + if self._must_cancel: + # replace status + info[0] = 'cancelling' + + coro = coroutines._format_coroutine(self._coro) + info.insert(1, 'coro=<%s>' % coro) + + if self._fut_waiter is not None: + info.insert(2, 'wait_for=%r' % self._fut_waiter) + return info + + def get_stack(self, *, limit=None): + """Return the list of stack frames for this task's coroutine. + + If the coroutine is not done, this returns the stack where it is + suspended. If the coroutine has completed successfully or was + cancelled, this returns an empty list. If the coroutine was + terminated by an exception, this returns the list of traceback + frames. + + The frames are always ordered from oldest to newest. + + The optional limit gives the maximum number of frames to + return; by default all available frames are returned. Its + meaning differs depending on whether a stack or a traceback is + returned: the newest frames of a stack are returned, but the + oldest frames of a traceback are returned. (This matches the + behavior of the traceback module.) + + For reasons beyond our control, only one stack frame is + returned for a suspended coroutine. + """ + frames = [] + try: + # 'async def' coroutines + f = self._coro.cr_frame + except AttributeError: + f = self._coro.gi_frame + if f is not None: + while f is not None: + if limit is not None: + if limit <= 0: + break + limit -= 1 + frames.append(f) + f = f.f_back + frames.reverse() + elif self._exception is not None: + tb = self._exception.__traceback__ + while tb is not None: + if limit is not None: + if limit <= 0: + break + limit -= 1 + frames.append(tb.tb_frame) + tb = tb.tb_next + return frames + + def print_stack(self, *, limit=None, file=None): + """Print the stack or traceback for this task's coroutine. + + This produces output similar to that of the traceback module, + for the frames retrieved by get_stack(). The limit argument + is passed to get_stack(). The file argument is an I/O stream + to which the output is written; by default output is written + to sys.stderr. + """ + extracted_list = [] + checked = set() + for f in self.get_stack(limit=limit): + lineno = f.f_lineno + co = f.f_code + filename = co.co_filename + name = co.co_name + if filename not in checked: + checked.add(filename) + linecache.checkcache(filename) + line = linecache.getline(filename, lineno, f.f_globals) + extracted_list.append((filename, lineno, name, line)) + exc = self._exception + if not extracted_list: + print('No stack for %r' % self, file=file) + elif exc is not None: + print('Traceback for %r (most recent call last):' % self, + file=file) + else: + print('Stack for %r (most recent call last):' % self, + file=file) + traceback.print_list(extracted_list, file=file) + if exc is not None: + for line in traceback.format_exception_only(exc.__class__, exc): + print(line, file=file, end='') + + def cancel(self): + """Request that this task cancel itself. + + This arranges for a CancelledError to be thrown into the + wrapped coroutine on the next cycle through the event loop. + The coroutine then has a chance to clean up or even deny + the request using try/except/finally. + + Unlike Future.cancel, this does not guarantee that the + task will be cancelled: the exception might be caught and + acted upon, delaying cancellation of the task or preventing + cancellation completely. The task may also return a value or + raise a different exception. + + Immediately after this method is called, Task.cancelled() will + not return True (unless the task was already cancelled). A + task will be marked as cancelled when the wrapped coroutine + terminates with a CancelledError exception (even if cancel() + was not called). + """ + if self.done(): + return False + if self._fut_waiter is not None: + if self._fut_waiter.cancel(): + # Leave self._fut_waiter; it may be a Task that + # catches and ignores the cancellation so we may have + # to cancel it again later. + return True + # It must be the case that self._step is already scheduled. + self._must_cancel = True + return True + + def _step(self, exc=None): + assert not self.done(), \ + '_step(): already done: {!r}, {!r}'.format(self, exc) + if self._must_cancel: + if not isinstance(exc, futures.CancelledError): + exc = futures.CancelledError() + self._must_cancel = False + coro = self._coro + self._fut_waiter = None + + self.__class__._current_tasks[self._loop] = self + # Call either coro.throw(exc) or coro.send(None). + try: + if exc is None: + # We use the `send` method directly, because coroutines + # don't have `__iter__` and `__next__` methods. + result = coro.send(None) + else: + result = coro.throw(exc) + except StopIteration as exc: + self.set_result(exc.value) + except futures.CancelledError: + super().cancel() # I.e., Future.cancel(self). + except Exception as exc: + self.set_exception(exc) + except BaseException as exc: + self.set_exception(exc) + raise + else: + blocking = getattr(result, '_asyncio_future_blocking', None) + if blocking is not None: + # Yielded Future must come from Future.__iter__(). + if result._loop is not self._loop: + self._loop.call_soon( + self._step, + RuntimeError( + 'Task {!r} got Future {!r} attached to a ' + 'different loop'.format(self, result))) + elif blocking: + if result is self: + self._loop.call_soon( + self._step, + RuntimeError( + 'Task cannot await on itself: {!r}'.format( + self))) + else: + result._asyncio_future_blocking = False + result.add_done_callback(self._wakeup) + self._fut_waiter = result + if self._must_cancel: + if self._fut_waiter.cancel(): + self._must_cancel = False + else: + self._loop.call_soon( + self._step, + RuntimeError( + 'yield was used instead of yield from ' + 'in task {!r} with {!r}'.format(self, result))) + elif result is None: + # Bare yield relinquishes control for one event loop iteration. + self._loop.call_soon(self._step) + elif inspect.isgenerator(result): + # Yielding a generator is just wrong. + self._loop.call_soon( + self._step, + RuntimeError( + 'yield was used instead of yield from for ' + 'generator in task {!r} with {}'.format( + self, result))) + else: + # Yielding something else is an error. + self._loop.call_soon( + self._step, + RuntimeError( + 'Task got bad yield: {!r}'.format(result))) + finally: + self.__class__._current_tasks.pop(self._loop) + self = None # Needed to break cycles when an exception occurs. + + def _wakeup(self, future): + try: + future.result() + except Exception as exc: + # This may also be a cancellation. + self._step(exc) + else: + # Don't pass the value of `future.result()` explicitly, + # as `Future.__iter__` and `Future.__await__` don't need it. + # If we call `_step(value, None)` instead of `_step()`, + # Python eval loop would use `.send(value)` method call, + # instead of `__next__()`, which is slower for futures + # that return non-generator iterators from their `__iter__`. + self._step() + self = None # Needed to break cycles when an exception occurs. + + +# wait() and as_completed() similar to those in PEP 3148. + +FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED +FIRST_EXCEPTION = concurrent.futures.FIRST_EXCEPTION +ALL_COMPLETED = concurrent.futures.ALL_COMPLETED + + +@coroutine +def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): + """Wait for the Futures and coroutines given by fs to complete. + + The sequence futures must not be empty. + + Coroutines will be wrapped in Tasks. + + Returns two sets of Future: (done, pending). + + Usage: + + done, pending = yield from asyncio.wait(fs) + + Note: This does not raise TimeoutError! Futures that aren't done + when the timeout occurs are returned in the second set. + """ + if futures.isfuture(fs) or coroutines.iscoroutine(fs): + raise TypeError("expect a list of futures, not %s" % type(fs).__name__) + if not fs: + raise ValueError('Set of coroutines/Futures is empty.') + if return_when not in (FIRST_COMPLETED, FIRST_EXCEPTION, ALL_COMPLETED): + raise ValueError('Invalid return_when value: {}'.format(return_when)) + + if loop is None: + loop = events.get_event_loop() + + fs = {ensure_future(f, loop=loop) for f in set(fs)} + + return (yield from _wait(fs, timeout, return_when, loop)) + + +def _release_waiter(waiter, *args): + if not waiter.done(): + waiter.set_result(None) + + +@coroutine +def wait_for(fut, timeout, *, loop=None): + """Wait for the single Future or coroutine to complete, with timeout. + + Coroutine will be wrapped in Task. + + Returns result of the Future or coroutine. When a timeout occurs, + it cancels the task and raises TimeoutError. To avoid the task + cancellation, wrap it in shield(). + + If the wait is cancelled, the task is also cancelled. + + This function is a coroutine. + """ + if loop is None: + loop = events.get_event_loop() + + if timeout is None: + return (yield from fut) + + waiter = loop.create_future() + timeout_handle = loop.call_later(timeout, _release_waiter, waiter) + cb = functools.partial(_release_waiter, waiter) + + fut = ensure_future(fut, loop=loop) + fut.add_done_callback(cb) + + try: + # wait until the future completes or the timeout + try: + yield from waiter + except futures.CancelledError: + fut.remove_done_callback(cb) + fut.cancel() + raise + + if fut.done(): + return fut.result() + else: + fut.remove_done_callback(cb) + fut.cancel() + raise futures.TimeoutError() + finally: + timeout_handle.cancel() + + +@coroutine +def _wait(fs, timeout, return_when, loop): + """Internal helper for wait() and wait_for(). + + The fs argument must be a collection of Futures. + """ + assert fs, 'Set of Futures is empty.' + waiter = loop.create_future() + timeout_handle = None + if timeout is not None: + timeout_handle = loop.call_later(timeout, _release_waiter, waiter) + counter = len(fs) + + def _on_completion(f): + nonlocal counter + counter -= 1 + if (counter <= 0 or + return_when == FIRST_COMPLETED or + return_when == FIRST_EXCEPTION and (not f.cancelled() and + f.exception() is not None)): + if timeout_handle is not None: + timeout_handle.cancel() + if not waiter.done(): + waiter.set_result(None) + + for f in fs: + f.add_done_callback(_on_completion) + + try: + yield from waiter + finally: + if timeout_handle is not None: + timeout_handle.cancel() + + done, pending = set(), set() + for f in fs: + f.remove_done_callback(_on_completion) + if f.done(): + done.add(f) + else: + pending.add(f) + return done, pending + + +# This is *not* a @coroutine! It is just an iterator (yielding Futures). +def as_completed(fs, *, loop=None, timeout=None): + """Return an iterator whose values are coroutines. + + When waiting for the yielded coroutines you'll get the results (or + exceptions!) of the original Futures (or coroutines), in the order + in which and as soon as they complete. + + This differs from PEP 3148; the proper way to use this is: + + for f in as_completed(fs): + result = yield from f # The 'yield from' may raise. + # Use result. + + If a timeout is specified, the 'yield from' will raise + TimeoutError when the timeout occurs before all Futures are done. + + Note: The futures 'f' are not necessarily members of fs. + """ + if futures.isfuture(fs) or coroutines.iscoroutine(fs): + raise TypeError("expect a list of futures, not %s" % type(fs).__name__) + loop = loop if loop is not None else events.get_event_loop() + todo = {ensure_future(f, loop=loop) for f in set(fs)} + from .queues import Queue # Import here to avoid circular import problem. + done = Queue(loop=loop) + timeout_handle = None + + def _on_timeout(): + for f in todo: + f.remove_done_callback(_on_completion) + done.put_nowait(None) # Queue a dummy value for _wait_for_one(). + todo.clear() # Can't do todo.remove(f) in the loop. + + def _on_completion(f): + if not todo: + return # _on_timeout() was here first. + todo.remove(f) + done.put_nowait(f) + if not todo and timeout_handle is not None: + timeout_handle.cancel() + + @coroutine + def _wait_for_one(): + f = yield from done.get() + if f is None: + # Dummy value from _on_timeout(). + raise futures.TimeoutError + return f.result() # May raise f.exception(). + + for f in todo: + f.add_done_callback(_on_completion) + if todo and timeout is not None: + timeout_handle = loop.call_later(timeout, _on_timeout) + for _ in range(len(todo)): + yield _wait_for_one() + + +@coroutine +def sleep(delay, result=None, *, loop=None): + """Coroutine that completes after a given time (in seconds).""" + if delay == 0: + yield + return result + + if loop is None: + loop = events.get_event_loop() + future = loop.create_future() + h = future._loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) + try: + return (yield from future) + finally: + h.cancel() + + +def async_(coro_or_future, *, loop=None): + """Wrap a coroutine in a future. + + If the argument is a Future, it is returned directly. + + This function is deprecated in 3.5. Use asyncio.ensure_future() instead. + """ + + warnings.warn("asyncio.async() function is deprecated, use ensure_future()", + DeprecationWarning) + + return ensure_future(coro_or_future, loop=loop) + +# Silence DeprecationWarning: +globals()['async'] = async_ +async_.__name__ = 'async' +del async_ + + +def ensure_future(coro_or_future, *, loop=None): + """Wrap a coroutine or an awaitable in a future. + + If the argument is a Future, it is returned directly. + """ + if futures.isfuture(coro_or_future): + if loop is not None and loop is not coro_or_future._loop: + raise ValueError('loop argument must agree with Future') + return coro_or_future + elif coroutines.iscoroutine(coro_or_future): + if loop is None: + loop = events.get_event_loop() + task = loop.create_task(coro_or_future) + if task._source_traceback: + del task._source_traceback[-1] + return task + elif compat.PY35 and inspect.isawaitable(coro_or_future): + return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) + else: + raise TypeError('A Future, a coroutine or an awaitable is required') + + +@coroutine +def _wrap_awaitable(awaitable): + """Helper for asyncio.ensure_future(). + + Wraps awaitable (an object with __await__) into a coroutine + that will later be wrapped in a Task by ensure_future(). + """ + return (yield from awaitable.__await__()) + + +class _GatheringFuture(futures.Future): + """Helper for gather(). + + This overrides cancel() to cancel all the children and act more + like Task.cancel(), which doesn't immediately mark itself as + cancelled. + """ + + def __init__(self, children, *, loop=None): + super().__init__(loop=loop) + self._children = children + + def cancel(self): + if self.done(): + return False + ret = False + for child in self._children: + if child.cancel(): + ret = True + return ret + + +def gather(*coros_or_futures, loop=None, return_exceptions=False): + """Return a future aggregating results from the given coroutines + or futures. + + Coroutines will be wrapped in a future and scheduled in the event + loop. They will not necessarily be scheduled in the same order as + passed in. + + All futures must share the same event loop. If all the tasks are + done successfully, the returned future's result is the list of + results (in the order of the original sequence, not necessarily + the order of results arrival). If *return_exceptions* is True, + exceptions in the tasks are treated the same as successful + results, and gathered in the result list; otherwise, the first + raised exception will be immediately propagated to the returned + future. + + Cancellation: if the outer Future is cancelled, all children (that + have not completed yet) are also cancelled. If any child is + cancelled, this is treated as if it raised CancelledError -- + the outer Future is *not* cancelled in this case. (This is to + prevent the cancellation of one child to cause other children to + be cancelled.) + """ + if not coros_or_futures: + if loop is None: + loop = events.get_event_loop() + outer = loop.create_future() + outer.set_result([]) + return outer + + arg_to_fut = {} + for arg in set(coros_or_futures): + if not futures.isfuture(arg): + fut = ensure_future(arg, loop=loop) + if loop is None: + loop = fut._loop + # The caller cannot control this future, the "destroy pending task" + # warning should not be emitted. + fut._log_destroy_pending = False + else: + fut = arg + if loop is None: + loop = fut._loop + elif fut._loop is not loop: + raise ValueError("futures are tied to different event loops") + arg_to_fut[arg] = fut + + children = [arg_to_fut[arg] for arg in coros_or_futures] + nchildren = len(children) + outer = _GatheringFuture(children, loop=loop) + nfinished = 0 + results = [None] * nchildren + + def _done_callback(i, fut): + nonlocal nfinished + if outer.done(): + if not fut.cancelled(): + # Mark exception retrieved. + fut.exception() + return + + if fut.cancelled(): + res = futures.CancelledError() + if not return_exceptions: + outer.set_exception(res) + return + elif fut._exception is not None: + res = fut.exception() # Mark exception retrieved. + if not return_exceptions: + outer.set_exception(res) + return + else: + res = fut._result + results[i] = res + nfinished += 1 + if nfinished == nchildren: + outer.set_result(results) + + for i, fut in enumerate(children): + fut.add_done_callback(functools.partial(_done_callback, i)) + return outer + + +def shield(arg, *, loop=None): + """Wait for a future, shielding it from cancellation. + + The statement + + res = yield from shield(something()) + + is exactly equivalent to the statement + + res = yield from something() + + *except* that if the coroutine containing it is cancelled, the + task running in something() is not cancelled. From the POV of + something(), the cancellation did not happen. But its caller is + still cancelled, so the yield-from expression still raises + CancelledError. Note: If something() is cancelled by other means + this will still cancel shield(). + + If you want to completely ignore cancellation (not recommended) + you can combine shield() with a try/except clause, as follows: + + try: + res = yield from shield(something()) + except CancelledError: + res = None + """ + inner = ensure_future(arg, loop=loop) + if inner.done(): + # Shortcut. + return inner + loop = inner._loop + outer = loop.create_future() + + def _done_callback(inner): + if outer.cancelled(): + if not inner.cancelled(): + # Mark inner's result as retrieved. + inner.exception() + return + + if inner.cancelled(): + outer.cancel() + else: + exc = inner.exception() + if exc is not None: + outer.set_exception(exc) + else: + outer.set_result(inner.result()) + + inner.add_done_callback(_done_callback) + return outer + + +def run_coroutine_threadsafe(coro, loop): + """Submit a coroutine object to a given event loop. + + Return a concurrent.futures.Future to access the result. + """ + if not coroutines.iscoroutine(coro): + raise TypeError('A coroutine object is required') + future = concurrent.futures.Future() + + def callback(): + try: + futures._chain_future(ensure_future(coro, loop=loop), future) + except Exception as exc: + if future.set_running_or_notify_cancel(): + future.set_exception(exc) + raise + + loop.call_soon_threadsafe(callback) + return future diff --git a/thirdparty/asyncio/asyncio/test_support.py b/thirdparty/asyncio/asyncio/test_support.py new file mode 100644 index 0000000..ada0a57 --- /dev/null +++ b/thirdparty/asyncio/asyncio/test_support.py @@ -0,0 +1,403 @@ +# Subset of test.support from CPython 3.5, just what we need to run asyncio +# test suite. The code is copied from CPython 3.5 to not depend on the test +# module because it is rarely installed. + +# Ignore symbol TEST_HOME_DIR: test_events works without it + +import contextlib +import functools +import gc +import os +import platform +import re +import socket +import subprocess +import sys +import time +import unittest +import warnings + + +# A constant likely larger than the underlying OS pipe buffer size, to +# make writes blocking. +# Windows limit seems to be around 512 B, and many Unix kernels have a +# 64 KiB pipe buffer size or 16 * PAGE_SIZE: take a few megs to be sure. +# (see issue #17835 for a discussion of this number). +PIPE_MAX_SIZE = 4 * 1024 * 1024 + 1 + + +class Error(Exception): + """Base class for regression test exceptions.""" + + +class TestFailed(Error): + """Test failed.""" + + +def strip_python_stderr(stderr): + """Strip the stderr of a Python process from potential debug output + emitted by the interpreter. + + This will typically be run on the result of the communicate() method + of a subprocess.Popen object. + """ + stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?", b"", stderr).strip() + return stderr + + +# Executing the interpreter in a subprocess +def _assert_python(expected_success, *args, **env_vars): + if '__isolated' in env_vars: + isolated = env_vars.pop('__isolated') + else: + isolated = not env_vars + cmd_line = [sys.executable, '-X', 'faulthandler'] + if isolated and sys.version_info >= (3, 4): + # isolated mode: ignore Python environment variables, ignore user + # site-packages, and don't add the current directory to sys.path + cmd_line.append('-I') + elif not env_vars: + # ignore Python environment variables + cmd_line.append('-E') + # Need to preserve the original environment, for in-place testing of + # shared library builds. + env = os.environ.copy() + # But a special flag that can be set to override -- in this case, the + # caller is responsible to pass the full environment. + if env_vars.pop('__cleanenv', None): + env = {} + env.update(env_vars) + cmd_line.extend(args) + p = subprocess.Popen(cmd_line, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + try: + out, err = p.communicate() + finally: + subprocess._cleanup() + p.stdout.close() + p.stderr.close() + rc = p.returncode + err = strip_python_stderr(err) + if (rc and expected_success) or (not rc and not expected_success): + raise AssertionError( + "Process return code is %d, " + "stderr follows:\n%s" % (rc, err.decode('ascii', 'ignore'))) + return rc, out, err + + +def assert_python_ok(*args, **env_vars): + """ + Assert that running the interpreter with `args` and optional environment + variables `env_vars` succeeds (rc == 0) and return a (return code, stdout, + stderr) tuple. + + If the __cleanenv keyword is set, env_vars is used a fresh environment. + + Python is started in isolated mode (command line option -I), + except if the __isolated keyword is set to False. + """ + return _assert_python(True, *args, **env_vars) + + +is_jython = sys.platform.startswith('java') + +def gc_collect(): + """Force as many objects as possible to be collected. + + In non-CPython implementations of Python, this is needed because timely + deallocation is not guaranteed by the garbage collector. (Even in CPython + this can be the case in case of reference cycles.) This means that __del__ + methods may be called later than expected and weakrefs may remain alive for + longer than expected. This function tries its best to force all garbage + objects to disappear. + """ + gc.collect() + if is_jython: + time.sleep(0.1) + gc.collect() + gc.collect() + + +HOST = "127.0.0.1" +HOSTv6 = "::1" + + +def _is_ipv6_enabled(): + """Check whether IPv6 is enabled on this host.""" + if socket.has_ipv6: + sock = None + try: + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.bind((HOSTv6, 0)) + return True + except OSError: + pass + finally: + if sock: + sock.close() + return False + +IPV6_ENABLED = _is_ipv6_enabled() + + +def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): + """Returns an unused port that should be suitable for binding. This is + achieved by creating a temporary socket with the same family and type as + the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to + the specified host address (defaults to 0.0.0.0) with the port set to 0, + eliciting an unused ephemeral port from the OS. The temporary socket is + then closed and deleted, and the ephemeral port is returned. + + Either this method or bind_port() should be used for any tests where a + server socket needs to be bound to a particular port for the duration of + the test. Which one to use depends on whether the calling code is creating + a python socket, or if an unused port needs to be provided in a constructor + or passed to an external program (i.e. the -accept argument to openssl's + s_server mode). Always prefer bind_port() over find_unused_port() where + possible. Hard coded ports should *NEVER* be used. As soon as a server + socket is bound to a hard coded port, the ability to run multiple instances + of the test simultaneously on the same host is compromised, which makes the + test a ticking time bomb in a buildbot environment. On Unix buildbots, this + may simply manifest as a failed test, which can be recovered from without + intervention in most cases, but on Windows, the entire python process can + completely and utterly wedge, requiring someone to log in to the buildbot + and manually kill the affected process. + + (This is easy to reproduce on Windows, unfortunately, and can be traced to + the SO_REUSEADDR socket option having different semantics on Windows versus + Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind, + listen and then accept connections on identical host/ports. An EADDRINUSE + OSError will be raised at some point (depending on the platform and + the order bind and listen were called on each socket). + + However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE + will ever be raised when attempting to bind two identical host/ports. When + accept() is called on each socket, the second caller's process will steal + the port from the first caller, leaving them both in an awkwardly wedged + state where they'll no longer respond to any signals or graceful kills, and + must be forcibly killed via OpenProcess()/TerminateProcess(). + + The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option + instead of SO_REUSEADDR, which effectively affords the same semantics as + SO_REUSEADDR on Unix. Given the propensity of Unix developers in the Open + Source world compared to Windows ones, this is a common mistake. A quick + look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when + openssl.exe is called with the 's_server' option, for example. See + http://bugs.python.org/issue2550 for more info. The following site also + has a very thorough description about the implications of both REUSEADDR + and EXCLUSIVEADDRUSE on Windows: + http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx) + + XXX: although this approach is a vast improvement on previous attempts to + elicit unused ports, it rests heavily on the assumption that the ephemeral + port returned to us by the OS won't immediately be dished back out to some + other process when we close and delete our temporary socket but before our + calling code has a chance to bind the returned port. We can deal with this + issue if/when we come across it. + """ + + tempsock = socket.socket(family, socktype) + port = bind_port(tempsock) + tempsock.close() + del tempsock + return port + +def bind_port(sock, host=HOST): + """Bind the socket to a free port and return the port number. Relies on + ephemeral ports in order to ensure we are using an unbound port. This is + important as many tests may be running simultaneously, especially in a + buildbot environment. This method raises an exception if the sock.family + is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR + or SO_REUSEPORT set on it. Tests should *never* set these socket options + for TCP/IP sockets. The only case for setting these options is testing + multicasting via multiple UDP sockets. + + Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e. + on Windows), it will be set on the socket. This will prevent anyone else + from bind()'ing to our host/port for the duration of the test. + """ + + if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM: + if hasattr(socket, 'SO_REUSEADDR'): + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1: + raise TestFailed("tests should never set the SO_REUSEADDR " + "socket option on TCP/IP sockets!") + if hasattr(socket, 'SO_REUSEPORT'): + try: + reuse = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) + if reuse == 1: + raise TestFailed("tests should never set the SO_REUSEPORT " + "socket option on TCP/IP sockets!") + except OSError: + # Python's socket module was compiled using modern headers + # thus defining SO_REUSEPORT but this process is running + # under an older kernel that does not support SO_REUSEPORT. + pass + if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) + + sock.bind((host, 0)) + port = sock.getsockname()[1] + return port + +def requires_mac_ver(*min_version): + """Decorator raising SkipTest if the OS is Mac OS X and the OS X + version if less than min_version. + + For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version + is lesser than 10.5. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if sys.platform == 'darwin': + version_txt = platform.mac_ver()[0] + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + pass + else: + if version < min_version: + min_version_txt = '.'.join(map(str, min_version)) + raise unittest.SkipTest( + "Mac OS X %s or higher required, not %s" + % (min_version_txt, version_txt)) + return func(*args, **kw) + wrapper.min_version = min_version + return wrapper + return decorator + +def _requires_unix_version(sysname, min_version): + """Decorator raising SkipTest if the OS is `sysname` and the version is + less than `min_version`. + + For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if + the FreeBSD version is less than 7.2. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if platform.system() == sysname: + version_txt = platform.release().split('-', 1)[0] + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + pass + else: + if version < min_version: + min_version_txt = '.'.join(map(str, min_version)) + raise unittest.SkipTest( + "%s version %s or higher required, not %s" + % (sysname, min_version_txt, version_txt)) + return func(*args, **kw) + wrapper.min_version = min_version + return wrapper + return decorator + +def requires_freebsd_version(*min_version): + """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version + is less than `min_version`. + + For example, @requires_freebsd_version(7, 2) raises SkipTest if the FreeBSD + version is less than 7.2. + """ + return _requires_unix_version('FreeBSD', min_version) + +class WarningsRecorder(object): + """Convenience wrapper for the warnings list returned on + entry to the warnings.catch_warnings() context manager. + """ + def __init__(self, warnings_list): + self._warnings = warnings_list + self._last = 0 + + def __getattr__(self, attr): + if len(self._warnings) > self._last: + return getattr(self._warnings[-1], attr) + elif attr in warnings.WarningMessage._WARNING_DETAILS: + return None + raise AttributeError("%r has no attribute %r" % (self, attr)) + + @property + def warnings(self): + return self._warnings[self._last:] + + def reset(self): + self._last = len(self._warnings) + +def _filterwarnings(filters, quiet=False): + """Catch the warnings, then check if all the expected + warnings have been raised and re-raise unexpected warnings. + If 'quiet' is True, only re-raise the unexpected warnings. + """ + # Clear the warning registry of the calling module + # in order to re-raise the warnings. + frame = sys._getframe(2) + registry = frame.f_globals.get('__warningregistry__') + if registry: + registry.clear() + with warnings.catch_warnings(record=True) as w: + # Set filter "always" to record all warnings. Because + # test_warnings swap the module, we need to look up in + # the sys.modules dictionary. + sys.modules['warnings'].simplefilter("always") + yield WarningsRecorder(w) + # Filter the recorded warnings + reraise = list(w) + missing = [] + for msg, cat in filters: + seen = False + for w in reraise[:]: + warning = w.message + # Filter out the matching messages + if (re.match(msg, str(warning), re.I) and + issubclass(warning.__class__, cat)): + seen = True + reraise.remove(w) + if not seen and not quiet: + # This filter caught nothing + missing.append((msg, cat.__name__)) + if reraise: + raise AssertionError("unhandled warning %s" % reraise[0]) + if missing: + raise AssertionError("filter (%r, %s) did not catch any warning" % + missing[0]) + +@contextlib.contextmanager +def check_warnings(*filters, **kwargs): + """Context manager to silence warnings. + + Accept 2-tuples as positional arguments: + ("message regexp", WarningCategory) + + Optional argument: + - if 'quiet' is True, it does not fail if a filter catches nothing + (default True without argument, + default False if some filters are defined) + + Without argument, it defaults to: + check_warnings(("", Warning), quiet=True) + """ + quiet = kwargs.get('quiet') + if not filters: + filters = (("", Warning),) + # Preserve backward compatibility + if quiet is None: + quiet = True + return _filterwarnings(filters, quiet) + +# Use test.support if available +try: + from test.support import * +except ImportError: + pass + +# Use test.script_helper if available +try: + from test.support.script_helper import assert_python_ok +except ImportError: + try: + from test.script_helper import assert_python_ok + except ImportError: + pass diff --git a/thirdparty/asyncio/asyncio/test_utils.py b/thirdparty/asyncio/asyncio/test_utils.py new file mode 100644 index 0000000..9d32822 --- /dev/null +++ b/thirdparty/asyncio/asyncio/test_utils.py @@ -0,0 +1,503 @@ +"""Utilities shared by tests.""" + +import collections +import contextlib +import io +import logging +import os +import re +import socket +import socketserver +import sys +import tempfile +import threading +import time +import unittest +import weakref + +from unittest import mock + +from http.server import HTTPServer +from wsgiref.simple_server import WSGIRequestHandler, WSGIServer + +try: + import ssl +except ImportError: # pragma: no cover + ssl = None + +from . import base_events +from . import compat +from . import events +from . import futures +from . import selectors +from . import tasks +from .coroutines import coroutine +from .log import logger + + +if sys.platform == 'win32': # pragma: no cover + from .windows_utils import socketpair +else: + from socket import socketpair # pragma: no cover + + +def dummy_ssl_context(): + if ssl is None: + return None + else: + return ssl.SSLContext(ssl.PROTOCOL_SSLv23) + + +def run_briefly(loop): + @coroutine + def once(): + pass + gen = once() + t = loop.create_task(gen) + # Don't log a warning if the task is not done after run_until_complete(). + # It occurs if the loop is stopped or if a task raises a BaseException. + t._log_destroy_pending = False + try: + loop.run_until_complete(t) + finally: + gen.close() + + +def run_until(loop, pred, timeout=30): + deadline = time.time() + timeout + while not pred(): + if timeout is not None: + timeout = deadline - time.time() + if timeout <= 0: + raise futures.TimeoutError() + loop.run_until_complete(tasks.sleep(0.001, loop=loop)) + + +def run_once(loop): + """Legacy API to run once through the event loop. + + This is the recommended pattern for test code. It will poll the + selector once and run all callbacks scheduled in response to I/O + events. + """ + loop.call_soon(loop.stop) + loop.run_forever() + + +class SilentWSGIRequestHandler(WSGIRequestHandler): + + def get_stderr(self): + return io.StringIO() + + def log_message(self, format, *args): + pass + + +class SilentWSGIServer(WSGIServer): + + request_timeout = 2 + + def get_request(self): + request, client_addr = super().get_request() + request.settimeout(self.request_timeout) + return request, client_addr + + def handle_error(self, request, client_address): + pass + + +class SSLWSGIServerMixin: + + def finish_request(self, request, client_address): + # The relative location of our test directory (which + # contains the ssl key and certificate files) differs + # between the stdlib and stand-alone asyncio. + # Prefer our own if we can find it. + here = os.path.join(os.path.dirname(__file__), '..', 'tests') + if not os.path.isdir(here): + here = os.path.join(os.path.dirname(os.__file__), + 'test', 'test_asyncio') + keyfile = os.path.join(here, 'ssl_key.pem') + certfile = os.path.join(here, 'ssl_cert.pem') + ssock = ssl.wrap_socket(request, + keyfile=keyfile, + certfile=certfile, + server_side=True) + try: + self.RequestHandlerClass(ssock, client_address, self) + ssock.close() + except OSError: + # maybe socket has been closed by peer + pass + + +class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer): + pass + + +def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls): + + def app(environ, start_response): + status = '200 OK' + headers = [('Content-type', 'text/plain')] + start_response(status, headers) + return [b'Test message'] + + # Run the test WSGI server in a separate thread in order not to + # interfere with event handling in the main thread + server_class = server_ssl_cls if use_ssl else server_cls + httpd = server_class(address, SilentWSGIRequestHandler) + httpd.set_app(app) + httpd.address = httpd.server_address + server_thread = threading.Thread( + target=lambda: httpd.serve_forever(poll_interval=0.05)) + server_thread.start() + try: + yield httpd + finally: + httpd.shutdown() + httpd.server_close() + server_thread.join() + + +if hasattr(socket, 'AF_UNIX'): + + class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer): + + def server_bind(self): + socketserver.UnixStreamServer.server_bind(self) + self.server_name = '127.0.0.1' + self.server_port = 80 + + + class UnixWSGIServer(UnixHTTPServer, WSGIServer): + + request_timeout = 2 + + def server_bind(self): + UnixHTTPServer.server_bind(self) + self.setup_environ() + + def get_request(self): + request, client_addr = super().get_request() + request.settimeout(self.request_timeout) + # Code in the stdlib expects that get_request + # will return a socket and a tuple (host, port). + # However, this isn't true for UNIX sockets, + # as the second return value will be a path; + # hence we return some fake data sufficient + # to get the tests going + return request, ('127.0.0.1', '') + + + class SilentUnixWSGIServer(UnixWSGIServer): + + def handle_error(self, request, client_address): + pass + + + class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer): + pass + + + def gen_unix_socket_path(): + with tempfile.NamedTemporaryFile() as file: + return file.name + + + @contextlib.contextmanager + def unix_socket_path(): + path = gen_unix_socket_path() + try: + yield path + finally: + try: + os.unlink(path) + except OSError: + pass + + + @contextlib.contextmanager + def run_test_unix_server(*, use_ssl=False): + with unix_socket_path() as path: + yield from _run_test_server(address=path, use_ssl=use_ssl, + server_cls=SilentUnixWSGIServer, + server_ssl_cls=UnixSSLWSGIServer) + + +@contextlib.contextmanager +def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False): + yield from _run_test_server(address=(host, port), use_ssl=use_ssl, + server_cls=SilentWSGIServer, + server_ssl_cls=SSLWSGIServer) + + +def make_test_protocol(base): + dct = {} + for name in dir(base): + if name.startswith('__') and name.endswith('__'): + # skip magic names + continue + dct[name] = MockCallback(return_value=None) + return type('TestProtocol', (base,) + base.__bases__, dct)() + + +class TestSelector(selectors.BaseSelector): + + def __init__(self): + self.keys = {} + + def register(self, fileobj, events, data=None): + key = selectors.SelectorKey(fileobj, 0, events, data) + self.keys[fileobj] = key + return key + + def unregister(self, fileobj): + return self.keys.pop(fileobj) + + def select(self, timeout): + return [] + + def get_map(self): + return self.keys + + +class TestLoop(base_events.BaseEventLoop): + """Loop for unittests. + + It manages self time directly. + If something scheduled to be executed later then + on next loop iteration after all ready handlers done + generator passed to __init__ is calling. + + Generator should be like this: + + def gen(): + ... + when = yield ... + ... = yield time_advance + + Value returned by yield is absolute time of next scheduled handler. + Value passed to yield is time advance to move loop's time forward. + """ + + def __init__(self, gen=None): + super().__init__() + + if gen is None: + def gen(): + yield + self._check_on_close = False + else: + self._check_on_close = True + + self._gen = gen() + next(self._gen) + self._time = 0 + self._clock_resolution = 1e-9 + self._timers = [] + self._selector = TestSelector() + + self.readers = {} + self.writers = {} + self.reset_counters() + + self._transports = weakref.WeakValueDictionary() + + def time(self): + return self._time + + def advance_time(self, advance): + """Move test time forward.""" + if advance: + self._time += advance + + def close(self): + super().close() + if self._check_on_close: + try: + self._gen.send(0) + except StopIteration: + pass + else: # pragma: no cover + raise AssertionError("Time generator is not finished") + + def _add_reader(self, fd, callback, *args): + self.readers[fd] = events.Handle(callback, args, self) + + def _remove_reader(self, fd): + self.remove_reader_count[fd] += 1 + if fd in self.readers: + del self.readers[fd] + return True + else: + return False + + def assert_reader(self, fd, callback, *args): + assert fd in self.readers, 'fd {} is not registered'.format(fd) + handle = self.readers[fd] + assert handle._callback == callback, '{!r} != {!r}'.format( + handle._callback, callback) + assert handle._args == args, '{!r} != {!r}'.format( + handle._args, args) + + def _add_writer(self, fd, callback, *args): + self.writers[fd] = events.Handle(callback, args, self) + + def _remove_writer(self, fd): + self.remove_writer_count[fd] += 1 + if fd in self.writers: + del self.writers[fd] + return True + else: + return False + + def assert_writer(self, fd, callback, *args): + assert fd in self.writers, 'fd {} is not registered'.format(fd) + handle = self.writers[fd] + assert handle._callback == callback, '{!r} != {!r}'.format( + handle._callback, callback) + assert handle._args == args, '{!r} != {!r}'.format( + handle._args, args) + + def _ensure_fd_no_transport(self, fd): + try: + transport = self._transports[fd] + except KeyError: + pass + else: + raise RuntimeError( + 'File descriptor {!r} is used by transport {!r}'.format( + fd, transport)) + + def add_reader(self, fd, callback, *args): + """Add a reader callback.""" + self._ensure_fd_no_transport(fd) + return self._add_reader(fd, callback, *args) + + def remove_reader(self, fd): + """Remove a reader callback.""" + self._ensure_fd_no_transport(fd) + return self._remove_reader(fd) + + def add_writer(self, fd, callback, *args): + """Add a writer callback..""" + self._ensure_fd_no_transport(fd) + return self._add_writer(fd, callback, *args) + + def remove_writer(self, fd): + """Remove a writer callback.""" + self._ensure_fd_no_transport(fd) + return self._remove_writer(fd) + + def reset_counters(self): + self.remove_reader_count = collections.defaultdict(int) + self.remove_writer_count = collections.defaultdict(int) + + def _run_once(self): + super()._run_once() + for when in self._timers: + advance = self._gen.send(when) + self.advance_time(advance) + self._timers = [] + + def call_at(self, when, callback, *args): + self._timers.append(when) + return super().call_at(when, callback, *args) + + def _process_events(self, event_list): + return + + def _write_to_self(self): + pass + + +def MockCallback(**kwargs): + return mock.Mock(spec=['__call__'], **kwargs) + + +class MockPattern(str): + """A regex based str with a fuzzy __eq__. + + Use this helper with 'mock.assert_called_with', or anywhere + where a regex comparison between strings is needed. + + For instance: + mock_call.assert_called_with(MockPattern('spam.*ham')) + """ + def __eq__(self, other): + return bool(re.search(str(self), other, re.S)) + + +def get_function_source(func): + source = events._get_function_source(func) + if source is None: + raise ValueError("unable to get the source of %r" % (func,)) + return source + + +class TestCase(unittest.TestCase): + def set_event_loop(self, loop, *, cleanup=True): + assert loop is not None + # ensure that the event loop is passed explicitly in asyncio + events.set_event_loop(None) + if cleanup: + self.addCleanup(loop.close) + + def new_test_loop(self, gen=None): + loop = TestLoop(gen) + self.set_event_loop(loop) + return loop + + def setUp(self): + self._get_running_loop = events._get_running_loop + events._get_running_loop = lambda: None + + def tearDown(self): + events._get_running_loop = self._get_running_loop + + events.set_event_loop(None) + + # Detect CPython bug #23353: ensure that yield/yield-from is not used + # in an except block of a generator + self.assertEqual(sys.exc_info(), (None, None, None)) + + if not compat.PY34: + # Python 3.3 compatibility + def subTest(self, *args, **kwargs): + class EmptyCM: + def __enter__(self): + pass + def __exit__(self, *exc): + pass + return EmptyCM() + + +@contextlib.contextmanager +def disable_logger(): + """Context manager to disable asyncio logger. + + For example, it can be used to ignore warnings in debug mode. + """ + old_level = logger.level + try: + logger.setLevel(logging.CRITICAL+1) + yield + finally: + logger.setLevel(old_level) + + +def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, + family=socket.AF_INET): + """Create a mock of a non-blocking socket.""" + sock = mock.MagicMock(socket.socket) + sock.proto = proto + sock.type = type + sock.family = family + sock.gettimeout.return_value = 0.0 + return sock + + +def force_legacy_ssl_support(): + return mock.patch('asyncio.sslproto._is_sslproto_available', + return_value=False) diff --git a/thirdparty/asyncio/asyncio/transports.py b/thirdparty/asyncio/asyncio/transports.py new file mode 100644 index 0000000..0db0875 --- /dev/null +++ b/thirdparty/asyncio/asyncio/transports.py @@ -0,0 +1,306 @@ +"""Abstract Transport class.""" + +from asyncio import compat + +__all__ = ['BaseTransport', 'ReadTransport', 'WriteTransport', + 'Transport', 'DatagramTransport', 'SubprocessTransport', + ] + + +class BaseTransport: + """Base class for transports.""" + + def __init__(self, extra=None): + if extra is None: + extra = {} + self._extra = extra + + def get_extra_info(self, name, default=None): + """Get optional transport information.""" + return self._extra.get(name, default) + + def is_closing(self): + """Return True if the transport is closing or closed.""" + raise NotImplementedError + + def close(self): + """Close the transport. + + Buffered data will be flushed asynchronously. No more data + will be received. After all buffered data is flushed, the + protocol's connection_lost() method will (eventually) called + with None as its argument. + """ + raise NotImplementedError + + def set_protocol(self, protocol): + """Set a new protocol.""" + raise NotImplementedError + + def get_protocol(self): + """Return the current protocol.""" + raise NotImplementedError + + +class ReadTransport(BaseTransport): + """Interface for read-only transports.""" + + def pause_reading(self): + """Pause the receiving end. + + No data will be passed to the protocol's data_received() + method until resume_reading() is called. + """ + raise NotImplementedError + + def resume_reading(self): + """Resume the receiving end. + + Data received will once again be passed to the protocol's + data_received() method. + """ + raise NotImplementedError + + +class WriteTransport(BaseTransport): + """Interface for write-only transports.""" + + def set_write_buffer_limits(self, high=None, low=None): + """Set the high- and low-water limits for write flow control. + + These two values control when to call the protocol's + pause_writing() and resume_writing() methods. If specified, + the low-water limit must be less than or equal to the + high-water limit. Neither value can be negative. + + The defaults are implementation-specific. If only the + high-water limit is given, the low-water limit defaults to an + implementation-specific value less than or equal to the + high-water limit. Setting high to zero forces low to zero as + well, and causes pause_writing() to be called whenever the + buffer becomes non-empty. Setting low to zero causes + resume_writing() to be called only once the buffer is empty. + Use of zero for either limit is generally sub-optimal as it + reduces opportunities for doing I/O and computation + concurrently. + """ + raise NotImplementedError + + def get_write_buffer_size(self): + """Return the current size of the write buffer.""" + raise NotImplementedError + + def write(self, data): + """Write some data bytes to the transport. + + This does not block; it buffers the data and arranges for it + to be sent out asynchronously. + """ + raise NotImplementedError + + def writelines(self, list_of_data): + """Write a list (or any iterable) of data bytes to the transport. + + The default implementation concatenates the arguments and + calls write() on the result. + """ + data = compat.flatten_list_bytes(list_of_data) + self.write(data) + + def write_eof(self): + """Close the write end after flushing buffered data. + + (This is like typing ^D into a UNIX program reading from stdin.) + + Data may still be received. + """ + raise NotImplementedError + + def can_write_eof(self): + """Return True if this transport supports write_eof(), False if not.""" + raise NotImplementedError + + def abort(self): + """Close the transport immediately. + + Buffered data will be lost. No more data will be received. + The protocol's connection_lost() method will (eventually) be + called with None as its argument. + """ + raise NotImplementedError + + +class Transport(ReadTransport, WriteTransport): + """Interface representing a bidirectional transport. + + There may be several implementations, but typically, the user does + not implement new transports; rather, the platform provides some + useful transports that are implemented using the platform's best + practices. + + The user never instantiates a transport directly; they call a + utility function, passing it a protocol factory and other + information necessary to create the transport and protocol. (E.g. + EventLoop.create_connection() or EventLoop.create_server().) + + The utility function will asynchronously create a transport and a + protocol and hook them up by calling the protocol's + connection_made() method, passing it the transport. + + The implementation here raises NotImplemented for every method + except writelines(), which calls write() in a loop. + """ + + +class DatagramTransport(BaseTransport): + """Interface for datagram (UDP) transports.""" + + def sendto(self, data, addr=None): + """Send data to the transport. + + This does not block; it buffers the data and arranges for it + to be sent out asynchronously. + addr is target socket address. + If addr is None use target address pointed on transport creation. + """ + raise NotImplementedError + + def abort(self): + """Close the transport immediately. + + Buffered data will be lost. No more data will be received. + The protocol's connection_lost() method will (eventually) be + called with None as its argument. + """ + raise NotImplementedError + + +class SubprocessTransport(BaseTransport): + + def get_pid(self): + """Get subprocess id.""" + raise NotImplementedError + + def get_returncode(self): + """Get subprocess returncode. + + See also + http://docs.python.org/3/library/subprocess#subprocess.Popen.returncode + """ + raise NotImplementedError + + def get_pipe_transport(self, fd): + """Get transport for pipe with number fd.""" + raise NotImplementedError + + def send_signal(self, signal): + """Send signal to subprocess. + + See also: + docs.python.org/3/library/subprocess#subprocess.Popen.send_signal + """ + raise NotImplementedError + + def terminate(self): + """Stop the subprocess. + + Alias for close() method. + + On Posix OSs the method sends SIGTERM to the subprocess. + On Windows the Win32 API function TerminateProcess() + is called to stop the subprocess. + + See also: + http://docs.python.org/3/library/subprocess#subprocess.Popen.terminate + """ + raise NotImplementedError + + def kill(self): + """Kill the subprocess. + + On Posix OSs the function sends SIGKILL to the subprocess. + On Windows kill() is an alias for terminate(). + + See also: + http://docs.python.org/3/library/subprocess#subprocess.Popen.kill + """ + raise NotImplementedError + + +class _FlowControlMixin(Transport): + """All the logic for (write) flow control in a mix-in base class. + + The subclass must implement get_write_buffer_size(). It must call + _maybe_pause_protocol() whenever the write buffer size increases, + and _maybe_resume_protocol() whenever it decreases. It may also + override set_write_buffer_limits() (e.g. to specify different + defaults). + + The subclass constructor must call super().__init__(extra). This + will call set_write_buffer_limits(). + + The user may call set_write_buffer_limits() and + get_write_buffer_size(), and their protocol's pause_writing() and + resume_writing() may be called. + """ + + def __init__(self, extra=None, loop=None): + super().__init__(extra) + assert loop is not None + self._loop = loop + self._protocol_paused = False + self._set_write_buffer_limits() + + def _maybe_pause_protocol(self): + size = self.get_write_buffer_size() + if size <= self._high_water: + return + if not self._protocol_paused: + self._protocol_paused = True + try: + self._protocol.pause_writing() + except Exception as exc: + self._loop.call_exception_handler({ + 'message': 'protocol.pause_writing() failed', + 'exception': exc, + 'transport': self, + 'protocol': self._protocol, + }) + + def _maybe_resume_protocol(self): + if (self._protocol_paused and + self.get_write_buffer_size() <= self._low_water): + self._protocol_paused = False + try: + self._protocol.resume_writing() + except Exception as exc: + self._loop.call_exception_handler({ + 'message': 'protocol.resume_writing() failed', + 'exception': exc, + 'transport': self, + 'protocol': self._protocol, + }) + + def get_write_buffer_limits(self): + return (self._low_water, self._high_water) + + def _set_write_buffer_limits(self, high=None, low=None): + if high is None: + if low is None: + high = 64*1024 + else: + high = 4*low + if low is None: + low = high // 4 + if not high >= low >= 0: + raise ValueError('high (%r) must be >= low (%r) must be >= 0' % + (high, low)) + self._high_water = high + self._low_water = low + + def set_write_buffer_limits(self, high=None, low=None): + self._set_write_buffer_limits(high=high, low=low) + self._maybe_pause_protocol() + + def get_write_buffer_size(self): + raise NotImplementedError diff --git a/thirdparty/asyncio/asyncio/unix_events.py b/thirdparty/asyncio/asyncio/unix_events.py new file mode 100644 index 0000000..65b61db --- /dev/null +++ b/thirdparty/asyncio/asyncio/unix_events.py @@ -0,0 +1,1064 @@ +"""Selector event loop for Unix with signal handling.""" + +import errno +import os +import signal +import socket +import stat +import subprocess +import sys +import threading +import warnings + + +from . import base_events +from . import base_subprocess +from . import compat +from . import constants +from . import coroutines +from . import events +from . import futures +from . import selector_events +from . import selectors +from . import transports +from .coroutines import coroutine +from .log import logger + + +__all__ = ['SelectorEventLoop', + 'AbstractChildWatcher', 'SafeChildWatcher', + 'FastChildWatcher', 'DefaultEventLoopPolicy', + ] + +if sys.platform == 'win32': # pragma: no cover + raise ImportError('Signals are not really supported on Windows') + + +def _sighandler_noop(signum, frame): + """Dummy signal handler.""" + pass + + +class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): + """Unix event loop. + + Adds signal handling and UNIX Domain Socket support to SelectorEventLoop. + """ + + def __init__(self, selector=None): + super().__init__(selector) + self._signal_handlers = {} + + def _socketpair(self): + return socket.socketpair() + + def close(self): + super().close() + for sig in list(self._signal_handlers): + self.remove_signal_handler(sig) + + def _process_self_data(self, data): + for signum in data: + if not signum: + # ignore null bytes written by _write_to_self() + continue + self._handle_signal(signum) + + def add_signal_handler(self, sig, callback, *args): + """Add a handler for a signal. UNIX only. + + Raise ValueError if the signal number is invalid or uncatchable. + Raise RuntimeError if there is a problem setting up the handler. + """ + if (coroutines.iscoroutine(callback) + or coroutines.iscoroutinefunction(callback)): + raise TypeError("coroutines cannot be used " + "with add_signal_handler()") + self._check_signal(sig) + self._check_closed() + try: + # set_wakeup_fd() raises ValueError if this is not the + # main thread. By calling it early we ensure that an + # event loop running in another thread cannot add a signal + # handler. + signal.set_wakeup_fd(self._csock.fileno()) + except (ValueError, OSError) as exc: + raise RuntimeError(str(exc)) + + handle = events.Handle(callback, args, self) + self._signal_handlers[sig] = handle + + try: + # Register a dummy signal handler to ask Python to write the signal + # number in the wakup file descriptor. _process_self_data() will + # read signal numbers from this file descriptor to handle signals. + signal.signal(sig, _sighandler_noop) + + # Set SA_RESTART to limit EINTR occurrences. + signal.siginterrupt(sig, False) + except OSError as exc: + del self._signal_handlers[sig] + if not self._signal_handlers: + try: + signal.set_wakeup_fd(-1) + except (ValueError, OSError) as nexc: + logger.info('set_wakeup_fd(-1) failed: %s', nexc) + + if exc.errno == errno.EINVAL: + raise RuntimeError('sig {} cannot be caught'.format(sig)) + else: + raise + + def _handle_signal(self, sig): + """Internal helper that is the actual signal handler.""" + handle = self._signal_handlers.get(sig) + if handle is None: + return # Assume it's some race condition. + if handle._cancelled: + self.remove_signal_handler(sig) # Remove it properly. + else: + self._add_callback_signalsafe(handle) + + def remove_signal_handler(self, sig): + """Remove a handler for a signal. UNIX only. + + Return True if a signal handler was removed, False if not. + """ + self._check_signal(sig) + try: + del self._signal_handlers[sig] + except KeyError: + return False + + if sig == signal.SIGINT: + handler = signal.default_int_handler + else: + handler = signal.SIG_DFL + + try: + signal.signal(sig, handler) + except OSError as exc: + if exc.errno == errno.EINVAL: + raise RuntimeError('sig {} cannot be caught'.format(sig)) + else: + raise + + if not self._signal_handlers: + try: + signal.set_wakeup_fd(-1) + except (ValueError, OSError) as exc: + logger.info('set_wakeup_fd(-1) failed: %s', exc) + + return True + + def _check_signal(self, sig): + """Internal helper to validate a signal. + + Raise ValueError if the signal number is invalid or uncatchable. + Raise RuntimeError if there is a problem setting up the handler. + """ + if not isinstance(sig, int): + raise TypeError('sig must be an int, not {!r}'.format(sig)) + + if not (1 <= sig < signal.NSIG): + raise ValueError( + 'sig {} out of range(1, {})'.format(sig, signal.NSIG)) + + def _make_read_pipe_transport(self, pipe, protocol, waiter=None, + extra=None): + return _UnixReadPipeTransport(self, pipe, protocol, waiter, extra) + + def _make_write_pipe_transport(self, pipe, protocol, waiter=None, + extra=None): + return _UnixWritePipeTransport(self, pipe, protocol, waiter, extra) + + @coroutine + def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): + with events.get_child_watcher() as watcher: + waiter = self.create_future() + transp = _UnixSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + + watcher.add_child_handler(transp.get_pid(), + self._child_watcher_callback, transp) + try: + yield from waiter + except Exception as exc: + # Workaround CPython bug #23353: using yield/yield-from in an + # except block of a generator doesn't clear properly + # sys.exc_info() + err = exc + else: + err = None + + if err is not None: + transp.close() + yield from transp._wait() + raise err + + return transp + + def _child_watcher_callback(self, pid, returncode, transp): + self.call_soon_threadsafe(transp._process_exited, returncode) + + @coroutine + def create_unix_connection(self, protocol_factory, path, *, + ssl=None, sock=None, + server_hostname=None): + assert server_hostname is None or isinstance(server_hostname, str) + if ssl: + if server_hostname is None: + raise ValueError( + 'you have to pass server_hostname when using ssl') + else: + if server_hostname is not None: + raise ValueError('server_hostname is only meaningful with ssl') + + if path is not None: + if sock is not None: + raise ValueError( + 'path and sock can not be specified at the same time') + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + try: + sock.setblocking(False) + yield from self.sock_connect(sock, path) + except: + sock.close() + raise + + else: + if sock is None: + raise ValueError('no path and sock were specified') + if (sock.family != socket.AF_UNIX or + sock.type != socket.SOCK_STREAM): + raise ValueError( + 'A UNIX Domain Stream Socket was expected, got {!r}' + .format(sock)) + sock.setblocking(False) + + transport, protocol = yield from self._create_connection_transport( + sock, protocol_factory, ssl, server_hostname) + return transport, protocol + + @coroutine + def create_unix_server(self, protocol_factory, path=None, *, + sock=None, backlog=100, ssl=None): + if isinstance(ssl, bool): + raise TypeError('ssl argument must be an SSLContext or None') + + if path is not None: + if sock is not None: + raise ValueError( + 'path and sock can not be specified at the same time') + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + # Check for abstract socket. `str` and `bytes` paths are supported. + if path[0] not in (0, '\x00'): + try: + if stat.S_ISSOCK(os.stat(path).st_mode): + os.remove(path) + except FileNotFoundError: + pass + except OSError as err: + # Directory may have permissions only to create socket. + logger.error('Unable to check or remove stale UNIX socket %r: %r', path, err) + + try: + sock.bind(path) + except OSError as exc: + sock.close() + if exc.errno == errno.EADDRINUSE: + # Let's improve the error message by adding + # with what exact address it occurs. + msg = 'Address {!r} is already in use'.format(path) + raise OSError(errno.EADDRINUSE, msg) from None + else: + raise + except: + sock.close() + raise + else: + if sock is None: + raise ValueError( + 'path was not specified, and no sock specified') + + if (sock.family != socket.AF_UNIX or + sock.type != socket.SOCK_STREAM): + raise ValueError( + 'A UNIX Domain Stream Socket was expected, got {!r}' + .format(sock)) + + server = base_events.Server(self, [sock]) + sock.listen(backlog) + sock.setblocking(False) + self._start_serving(protocol_factory, sock, ssl, server) + return server + + +if hasattr(os, 'set_blocking'): + def _set_nonblocking(fd): + os.set_blocking(fd, False) +else: + import fcntl + + def _set_nonblocking(fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + flags = flags | os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, flags) + + +class _UnixReadPipeTransport(transports.ReadTransport): + + max_size = 256 * 1024 # max bytes we read in one event loop iteration + + def __init__(self, loop, pipe, protocol, waiter=None, extra=None): + super().__init__(extra) + self._extra['pipe'] = pipe + self._loop = loop + self._pipe = pipe + self._fileno = pipe.fileno() + self._protocol = protocol + self._closing = False + + mode = os.fstat(self._fileno).st_mode + if not (stat.S_ISFIFO(mode) or + stat.S_ISSOCK(mode) or + stat.S_ISCHR(mode)): + self._pipe = None + self._fileno = None + self._protocol = None + raise ValueError("Pipe transport is for pipes/sockets only.") + + _set_nonblocking(self._fileno) + + self._loop.call_soon(self._protocol.connection_made, self) + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop._add_reader, + self._fileno, self._read_ready) + if waiter is not None: + # only wake up the waiter when connection_made() has been called + self._loop.call_soon(futures._set_result_unless_cancelled, + waiter, None) + + def __repr__(self): + info = [self.__class__.__name__] + if self._pipe is None: + info.append('closed') + elif self._closing: + info.append('closing') + info.append('fd=%s' % self._fileno) + selector = getattr(self._loop, '_selector', None) + if self._pipe is not None and selector is not None: + polling = selector_events._test_selector_event( + selector, + self._fileno, selectors.EVENT_READ) + if polling: + info.append('polling') + else: + info.append('idle') + elif self._pipe is not None: + info.append('open') + else: + info.append('closed') + return '<%s>' % ' '.join(info) + + def _read_ready(self): + try: + data = os.read(self._fileno, self.max_size) + except (BlockingIOError, InterruptedError): + pass + except OSError as exc: + self._fatal_error(exc, 'Fatal read error on pipe transport') + else: + if data: + self._protocol.data_received(data) + else: + if self._loop.get_debug(): + logger.info("%r was closed by peer", self) + self._closing = True + self._loop._remove_reader(self._fileno) + self._loop.call_soon(self._protocol.eof_received) + self._loop.call_soon(self._call_connection_lost, None) + + def pause_reading(self): + self._loop._remove_reader(self._fileno) + + def resume_reading(self): + self._loop._add_reader(self._fileno, self._read_ready) + + def set_protocol(self, protocol): + self._protocol = protocol + + def get_protocol(self): + return self._protocol + + def is_closing(self): + return self._closing + + def close(self): + if not self._closing: + self._close(None) + + # On Python 3.3 and older, objects with a destructor part of a reference + # cycle are never destroyed. It's not more the case on Python 3.4 thanks + # to the PEP 442. + if compat.PY34: + def __del__(self): + if self._pipe is not None: + warnings.warn("unclosed transport %r" % self, ResourceWarning) + self._pipe.close() + + def _fatal_error(self, exc, message='Fatal error on pipe transport'): + # should be called by exception handler only + if (isinstance(exc, OSError) and exc.errno == errno.EIO): + if self._loop.get_debug(): + logger.debug("%r: %s", self, message, exc_info=True) + else: + self._loop.call_exception_handler({ + 'message': message, + 'exception': exc, + 'transport': self, + 'protocol': self._protocol, + }) + self._close(exc) + + def _close(self, exc): + self._closing = True + self._loop._remove_reader(self._fileno) + self._loop.call_soon(self._call_connection_lost, exc) + + def _call_connection_lost(self, exc): + try: + self._protocol.connection_lost(exc) + finally: + self._pipe.close() + self._pipe = None + self._protocol = None + self._loop = None + + +class _UnixWritePipeTransport(transports._FlowControlMixin, + transports.WriteTransport): + + def __init__(self, loop, pipe, protocol, waiter=None, extra=None): + super().__init__(extra, loop) + self._extra['pipe'] = pipe + self._pipe = pipe + self._fileno = pipe.fileno() + self._protocol = protocol + self._buffer = bytearray() + self._conn_lost = 0 + self._closing = False # Set when close() or write_eof() called. + + mode = os.fstat(self._fileno).st_mode + is_char = stat.S_ISCHR(mode) + is_fifo = stat.S_ISFIFO(mode) + is_socket = stat.S_ISSOCK(mode) + if not (is_char or is_fifo or is_socket): + self._pipe = None + self._fileno = None + self._protocol = None + raise ValueError("Pipe transport is only for " + "pipes, sockets and character devices") + + _set_nonblocking(self._fileno) + self._loop.call_soon(self._protocol.connection_made, self) + + # On AIX, the reader trick (to be notified when the read end of the + # socket is closed) only works for sockets. On other platforms it + # works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.) + if is_socket or (is_fifo and not sys.platform.startswith("aix")): + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop._add_reader, + self._fileno, self._read_ready) + + if waiter is not None: + # only wake up the waiter when connection_made() has been called + self._loop.call_soon(futures._set_result_unless_cancelled, + waiter, None) + + def __repr__(self): + info = [self.__class__.__name__] + if self._pipe is None: + info.append('closed') + elif self._closing: + info.append('closing') + info.append('fd=%s' % self._fileno) + selector = getattr(self._loop, '_selector', None) + if self._pipe is not None and selector is not None: + polling = selector_events._test_selector_event( + selector, + self._fileno, selectors.EVENT_WRITE) + if polling: + info.append('polling') + else: + info.append('idle') + + bufsize = self.get_write_buffer_size() + info.append('bufsize=%s' % bufsize) + elif self._pipe is not None: + info.append('open') + else: + info.append('closed') + return '<%s>' % ' '.join(info) + + def get_write_buffer_size(self): + return len(self._buffer) + + def _read_ready(self): + # Pipe was closed by peer. + if self._loop.get_debug(): + logger.info("%r was closed by peer", self) + if self._buffer: + self._close(BrokenPipeError()) + else: + self._close() + + def write(self, data): + assert isinstance(data, (bytes, bytearray, memoryview)), repr(data) + if isinstance(data, bytearray): + data = memoryview(data) + if not data: + return + + if self._conn_lost or self._closing: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('pipe closed by peer or ' + 'os.write(pipe, data) raised exception.') + self._conn_lost += 1 + return + + if not self._buffer: + # Attempt to send it right away first. + try: + n = os.write(self._fileno, data) + except (BlockingIOError, InterruptedError): + n = 0 + except Exception as exc: + self._conn_lost += 1 + self._fatal_error(exc, 'Fatal write error on pipe transport') + return + if n == len(data): + return + elif n > 0: + data = memoryview(data)[n:] + self._loop._add_writer(self._fileno, self._write_ready) + + self._buffer += data + self._maybe_pause_protocol() + + def _write_ready(self): + assert self._buffer, 'Data should not be empty' + + try: + n = os.write(self._fileno, self._buffer) + except (BlockingIOError, InterruptedError): + pass + except Exception as exc: + self._buffer.clear() + self._conn_lost += 1 + # Remove writer here, _fatal_error() doesn't it + # because _buffer is empty. + self._loop._remove_writer(self._fileno) + self._fatal_error(exc, 'Fatal write error on pipe transport') + else: + if n == len(self._buffer): + self._buffer.clear() + self._loop._remove_writer(self._fileno) + self._maybe_resume_protocol() # May append to buffer. + if self._closing: + self._loop._remove_reader(self._fileno) + self._call_connection_lost(None) + return + elif n > 0: + del self._buffer[:n] + + def can_write_eof(self): + return True + + def write_eof(self): + if self._closing: + return + assert self._pipe + self._closing = True + if not self._buffer: + self._loop._remove_reader(self._fileno) + self._loop.call_soon(self._call_connection_lost, None) + + def set_protocol(self, protocol): + self._protocol = protocol + + def get_protocol(self): + return self._protocol + + def is_closing(self): + return self._closing + + def close(self): + if self._pipe is not None and not self._closing: + # write_eof is all what we needed to close the write pipe + self.write_eof() + + # On Python 3.3 and older, objects with a destructor part of a reference + # cycle are never destroyed. It's not more the case on Python 3.4 thanks + # to the PEP 442. + if compat.PY34: + def __del__(self): + if self._pipe is not None: + warnings.warn("unclosed transport %r" % self, ResourceWarning) + self._pipe.close() + + def abort(self): + self._close(None) + + def _fatal_error(self, exc, message='Fatal error on pipe transport'): + # should be called by exception handler only + if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if self._loop.get_debug(): + logger.debug("%r: %s", self, message, exc_info=True) + else: + self._loop.call_exception_handler({ + 'message': message, + 'exception': exc, + 'transport': self, + 'protocol': self._protocol, + }) + self._close(exc) + + def _close(self, exc=None): + self._closing = True + if self._buffer: + self._loop._remove_writer(self._fileno) + self._buffer.clear() + self._loop._remove_reader(self._fileno) + self._loop.call_soon(self._call_connection_lost, exc) + + def _call_connection_lost(self, exc): + try: + self._protocol.connection_lost(exc) + finally: + self._pipe.close() + self._pipe = None + self._protocol = None + self._loop = None + + +if hasattr(os, 'set_inheritable'): + # Python 3.4 and newer + _set_inheritable = os.set_inheritable +else: + import fcntl + + def _set_inheritable(fd, inheritable): + cloexec_flag = getattr(fcntl, 'FD_CLOEXEC', 1) + + old = fcntl.fcntl(fd, fcntl.F_GETFD) + if not inheritable: + fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) + + +class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): + + def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): + stdin_w = None + if stdin == subprocess.PIPE: + # Use a socket pair for stdin, since not all platforms + # support selecting read events on the write end of a + # socket (which we use in order to detect closing of the + # other end). Notably this is needed on AIX, and works + # just fine on other platforms. + stdin, stdin_w = self._loop._socketpair() + + # Mark the write end of the stdin pipe as non-inheritable, + # needed by close_fds=False on Python 3.3 and older + # (Python 3.4 implements the PEP 446, socketpair returns + # non-inheritable sockets) + _set_inheritable(stdin_w.fileno(), False) + self._proc = subprocess.Popen( + args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, + universal_newlines=False, bufsize=bufsize, **kwargs) + if stdin_w is not None: + stdin.close() + self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize) + + +class AbstractChildWatcher: + """Abstract base class for monitoring child processes. + + Objects derived from this class monitor a collection of subprocesses and + report their termination or interruption by a signal. + + New callbacks are registered with .add_child_handler(). Starting a new + process must be done within a 'with' block to allow the watcher to suspend + its activity until the new process if fully registered (this is needed to + prevent a race condition in some implementations). + + Example: + with watcher: + proc = subprocess.Popen("sleep 1") + watcher.add_child_handler(proc.pid, callback) + + Notes: + Implementations of this class must be thread-safe. + + Since child watcher objects may catch the SIGCHLD signal and call + waitpid(-1), there should be only one active object per process. + """ + + def add_child_handler(self, pid, callback, *args): + """Register a new child handler. + + Arrange for callback(pid, returncode, *args) to be called when + process 'pid' terminates. Specifying another callback for the same + process replaces the previous handler. + + Note: callback() must be thread-safe. + """ + raise NotImplementedError() + + def remove_child_handler(self, pid): + """Removes the handler for process 'pid'. + + The function returns True if the handler was successfully removed, + False if there was nothing to remove.""" + + raise NotImplementedError() + + def attach_loop(self, loop): + """Attach the watcher to an event loop. + + If the watcher was previously attached to an event loop, then it is + first detached before attaching to the new loop. + + Note: loop may be None. + """ + raise NotImplementedError() + + def close(self): + """Close the watcher. + + This must be called to make sure that any underlying resource is freed. + """ + raise NotImplementedError() + + def __enter__(self): + """Enter the watcher's context and allow starting new processes + + This function must return self""" + raise NotImplementedError() + + def __exit__(self, a, b, c): + """Exit the watcher's context""" + raise NotImplementedError() + + +class BaseChildWatcher(AbstractChildWatcher): + + def __init__(self): + self._loop = None + self._callbacks = {} + + def close(self): + self.attach_loop(None) + + def _do_waitpid(self, expected_pid): + raise NotImplementedError() + + def _do_waitpid_all(self): + raise NotImplementedError() + + def attach_loop(self, loop): + assert loop is None or isinstance(loop, events.AbstractEventLoop) + + if self._loop is not None and loop is None and self._callbacks: + warnings.warn( + 'A loop is being detached ' + 'from a child watcher with pending handlers', + RuntimeWarning) + + if self._loop is not None: + self._loop.remove_signal_handler(signal.SIGCHLD) + + self._loop = loop + if loop is not None: + loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) + + # Prevent a race condition in case a child terminated + # during the switch. + self._do_waitpid_all() + + def _sig_chld(self): + try: + self._do_waitpid_all() + except Exception as exc: + # self._loop should always be available here + # as '_sig_chld' is added as a signal handler + # in 'attach_loop' + self._loop.call_exception_handler({ + 'message': 'Unknown exception in SIGCHLD handler', + 'exception': exc, + }) + + def _compute_returncode(self, status): + if os.WIFSIGNALED(status): + # The child process died because of a signal. + return -os.WTERMSIG(status) + elif os.WIFEXITED(status): + # The child process exited (e.g sys.exit()). + return os.WEXITSTATUS(status) + else: + # The child exited, but we don't understand its status. + # This shouldn't happen, but if it does, let's just + # return that status; perhaps that helps debug it. + return status + + +class SafeChildWatcher(BaseChildWatcher): + """'Safe' child watcher implementation. + + This implementation avoids disrupting other code spawning processes by + polling explicitly each process in the SIGCHLD handler instead of calling + os.waitpid(-1). + + This is a safe solution but it has a significant overhead when handling a + big number of children (O(n) each time SIGCHLD is raised) + """ + + def close(self): + self._callbacks.clear() + super().close() + + def __enter__(self): + return self + + def __exit__(self, a, b, c): + pass + + def add_child_handler(self, pid, callback, *args): + if self._loop is None: + raise RuntimeError( + "Cannot add child handler, " + "the child watcher does not have a loop attached") + + self._callbacks[pid] = (callback, args) + + # Prevent a race condition in case the child is already terminated. + self._do_waitpid(pid) + + def remove_child_handler(self, pid): + try: + del self._callbacks[pid] + return True + except KeyError: + return False + + def _do_waitpid_all(self): + + for pid in list(self._callbacks): + self._do_waitpid(pid) + + def _do_waitpid(self, expected_pid): + assert expected_pid > 0 + + try: + pid, status = os.waitpid(expected_pid, os.WNOHANG) + except ChildProcessError: + # The child process is already reaped + # (may happen if waitpid() is called elsewhere). + pid = expected_pid + returncode = 255 + logger.warning( + "Unknown child process pid %d, will report returncode 255", + pid) + else: + if pid == 0: + # The child process is still alive. + return + + returncode = self._compute_returncode(status) + if self._loop.get_debug(): + logger.debug('process %s exited with returncode %s', + expected_pid, returncode) + + try: + callback, args = self._callbacks.pop(pid) + except KeyError: # pragma: no cover + # May happen if .remove_child_handler() is called + # after os.waitpid() returns. + if self._loop.get_debug(): + logger.warning("Child watcher got an unexpected pid: %r", + pid, exc_info=True) + else: + callback(pid, returncode, *args) + + +class FastChildWatcher(BaseChildWatcher): + """'Fast' child watcher implementation. + + This implementation reaps every terminated processes by calling + os.waitpid(-1) directly, possibly breaking other code spawning processes + and waiting for their termination. + + There is no noticeable overhead when handling a big number of children + (O(1) each time a child terminates). + """ + def __init__(self): + super().__init__() + self._lock = threading.Lock() + self._zombies = {} + self._forks = 0 + + def close(self): + self._callbacks.clear() + self._zombies.clear() + super().close() + + def __enter__(self): + with self._lock: + self._forks += 1 + + return self + + def __exit__(self, a, b, c): + with self._lock: + self._forks -= 1 + + if self._forks or not self._zombies: + return + + collateral_victims = str(self._zombies) + self._zombies.clear() + + logger.warning( + "Caught subprocesses termination from unknown pids: %s", + collateral_victims) + + def add_child_handler(self, pid, callback, *args): + assert self._forks, "Must use the context manager" + + if self._loop is None: + raise RuntimeError( + "Cannot add child handler, " + "the child watcher does not have a loop attached") + + with self._lock: + try: + returncode = self._zombies.pop(pid) + except KeyError: + # The child is running. + self._callbacks[pid] = callback, args + return + + # The child is dead already. We can fire the callback. + callback(pid, returncode, *args) + + def remove_child_handler(self, pid): + try: + del self._callbacks[pid] + return True + except KeyError: + return False + + def _do_waitpid_all(self): + # Because of signal coalescing, we must keep calling waitpid() as + # long as we're able to reap a child. + while True: + try: + pid, status = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + # No more child processes exist. + return + else: + if pid == 0: + # A child process is still alive. + return + + returncode = self._compute_returncode(status) + + with self._lock: + try: + callback, args = self._callbacks.pop(pid) + except KeyError: + # unknown child + if self._forks: + # It may not be registered yet. + self._zombies[pid] = returncode + if self._loop.get_debug(): + logger.debug('unknown process %s exited ' + 'with returncode %s', + pid, returncode) + continue + callback = None + else: + if self._loop.get_debug(): + logger.debug('process %s exited with returncode %s', + pid, returncode) + + if callback is None: + logger.warning( + "Caught subprocess termination from unknown pid: " + "%d -> %d", pid, returncode) + else: + callback(pid, returncode, *args) + + +class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): + """UNIX event loop policy with a watcher for child processes.""" + _loop_factory = _UnixSelectorEventLoop + + def __init__(self): + super().__init__() + self._watcher = None + + def _init_watcher(self): + with events._lock: + if self._watcher is None: # pragma: no branch + self._watcher = SafeChildWatcher() + if isinstance(threading.current_thread(), + threading._MainThread): + self._watcher.attach_loop(self._local._loop) + + def set_event_loop(self, loop): + """Set the event loop. + + As a side effect, if a child watcher was set before, then calling + .set_event_loop() from the main thread will call .attach_loop(loop) on + the child watcher. + """ + + super().set_event_loop(loop) + + if self._watcher is not None and \ + isinstance(threading.current_thread(), threading._MainThread): + self._watcher.attach_loop(loop) + + def get_child_watcher(self): + """Get the watcher for child processes. + + If not yet set, a SafeChildWatcher object is automatically created. + """ + if self._watcher is None: + self._init_watcher() + + return self._watcher + + def set_child_watcher(self, watcher): + """Set the watcher for child processes.""" + + assert watcher is None or isinstance(watcher, AbstractChildWatcher) + + if self._watcher is not None: + self._watcher.close() + + self._watcher = watcher + +SelectorEventLoop = _UnixSelectorEventLoop +DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy diff --git a/thirdparty/asyncio/asyncio/windows_events.py b/thirdparty/asyncio/asyncio/windows_events.py new file mode 100644 index 0000000..668fe14 --- /dev/null +++ b/thirdparty/asyncio/asyncio/windows_events.py @@ -0,0 +1,774 @@ +"""Selector and proactor event loops for Windows.""" + +import _winapi +import errno +import math +import socket +import struct +import weakref + +from . import events +from . import base_subprocess +from . import futures +from . import proactor_events +from . import selector_events +from . import tasks +from . import windows_utils +from . import _overlapped +from .coroutines import coroutine +from .log import logger + + +__all__ = ['SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', + 'DefaultEventLoopPolicy', + ] + + +NULL = 0 +INFINITE = 0xffffffff +ERROR_CONNECTION_REFUSED = 1225 +ERROR_CONNECTION_ABORTED = 1236 + +# Initial delay in seconds for connect_pipe() before retrying to connect +CONNECT_PIPE_INIT_DELAY = 0.001 + +# Maximum delay in seconds for connect_pipe() before retrying to connect +CONNECT_PIPE_MAX_DELAY = 0.100 + + +class _OverlappedFuture(futures.Future): + """Subclass of Future which represents an overlapped operation. + + Cancelling it will immediately cancel the overlapped operation. + """ + + def __init__(self, ov, *, loop=None): + super().__init__(loop=loop) + if self._source_traceback: + del self._source_traceback[-1] + self._ov = ov + + def _repr_info(self): + info = super()._repr_info() + if self._ov is not None: + state = 'pending' if self._ov.pending else 'completed' + info.insert(1, 'overlapped=<%s, %#x>' % (state, self._ov.address)) + return info + + def _cancel_overlapped(self): + if self._ov is None: + return + try: + self._ov.cancel() + except OSError as exc: + context = { + 'message': 'Cancelling an overlapped future failed', + 'exception': exc, + 'future': self, + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + self._ov = None + + def cancel(self): + self._cancel_overlapped() + return super().cancel() + + def set_exception(self, exception): + super().set_exception(exception) + self._cancel_overlapped() + + def set_result(self, result): + super().set_result(result) + self._ov = None + + +class _BaseWaitHandleFuture(futures.Future): + """Subclass of Future which represents a wait handle.""" + + def __init__(self, ov, handle, wait_handle, *, loop=None): + super().__init__(loop=loop) + if self._source_traceback: + del self._source_traceback[-1] + # Keep a reference to the Overlapped object to keep it alive until the + # wait is unregistered + self._ov = ov + self._handle = handle + self._wait_handle = wait_handle + + # Should we call UnregisterWaitEx() if the wait completes + # or is cancelled? + self._registered = True + + def _poll(self): + # non-blocking wait: use a timeout of 0 millisecond + return (_winapi.WaitForSingleObject(self._handle, 0) == + _winapi.WAIT_OBJECT_0) + + def _repr_info(self): + info = super()._repr_info() + info.append('handle=%#x' % self._handle) + if self._handle is not None: + state = 'signaled' if self._poll() else 'waiting' + info.append(state) + if self._wait_handle is not None: + info.append('wait_handle=%#x' % self._wait_handle) + return info + + def _unregister_wait_cb(self, fut): + # The wait was unregistered: it's not safe to destroy the Overlapped + # object + self._ov = None + + def _unregister_wait(self): + if not self._registered: + return + self._registered = False + + wait_handle = self._wait_handle + self._wait_handle = None + try: + _overlapped.UnregisterWait(wait_handle) + except OSError as exc: + if exc.winerror != _overlapped.ERROR_IO_PENDING: + context = { + 'message': 'Failed to unregister the wait handle', + 'exception': exc, + 'future': self, + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + return + # ERROR_IO_PENDING means that the unregister is pending + + self._unregister_wait_cb(None) + + def cancel(self): + self._unregister_wait() + return super().cancel() + + def set_exception(self, exception): + self._unregister_wait() + super().set_exception(exception) + + def set_result(self, result): + self._unregister_wait() + super().set_result(result) + + +class _WaitCancelFuture(_BaseWaitHandleFuture): + """Subclass of Future which represents a wait for the cancellation of a + _WaitHandleFuture using an event. + """ + + def __init__(self, ov, event, wait_handle, *, loop=None): + super().__init__(ov, event, wait_handle, loop=loop) + + self._done_callback = None + + def cancel(self): + raise RuntimeError("_WaitCancelFuture must not be cancelled") + + def _schedule_callbacks(self): + super(_WaitCancelFuture, self)._schedule_callbacks() + if self._done_callback is not None: + self._done_callback(self) + + +class _WaitHandleFuture(_BaseWaitHandleFuture): + def __init__(self, ov, handle, wait_handle, proactor, *, loop=None): + super().__init__(ov, handle, wait_handle, loop=loop) + self._proactor = proactor + self._unregister_proactor = True + self._event = _overlapped.CreateEvent(None, True, False, None) + self._event_fut = None + + def _unregister_wait_cb(self, fut): + if self._event is not None: + _winapi.CloseHandle(self._event) + self._event = None + self._event_fut = None + + # If the wait was cancelled, the wait may never be signalled, so + # it's required to unregister it. Otherwise, IocpProactor.close() will + # wait forever for an event which will never come. + # + # If the IocpProactor already received the event, it's safe to call + # _unregister() because we kept a reference to the Overlapped object + # which is used as a unique key. + self._proactor._unregister(self._ov) + self._proactor = None + + super()._unregister_wait_cb(fut) + + def _unregister_wait(self): + if not self._registered: + return + self._registered = False + + wait_handle = self._wait_handle + self._wait_handle = None + try: + _overlapped.UnregisterWaitEx(wait_handle, self._event) + except OSError as exc: + if exc.winerror != _overlapped.ERROR_IO_PENDING: + context = { + 'message': 'Failed to unregister the wait handle', + 'exception': exc, + 'future': self, + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + return + # ERROR_IO_PENDING is not an error, the wait was unregistered + + self._event_fut = self._proactor._wait_cancel(self._event, + self._unregister_wait_cb) + + +class PipeServer(object): + """Class representing a pipe server. + + This is much like a bound, listening socket. + """ + def __init__(self, address): + self._address = address + self._free_instances = weakref.WeakSet() + # initialize the pipe attribute before calling _server_pipe_handle() + # because this function can raise an exception and the destructor calls + # the close() method + self._pipe = None + self._accept_pipe_future = None + self._pipe = self._server_pipe_handle(True) + + def _get_unconnected_pipe(self): + # Create new instance and return previous one. This ensures + # that (until the server is closed) there is always at least + # one pipe handle for address. Therefore if a client attempt + # to connect it will not fail with FileNotFoundError. + tmp, self._pipe = self._pipe, self._server_pipe_handle(False) + return tmp + + def _server_pipe_handle(self, first): + # Return a wrapper for a new pipe handle. + if self.closed(): + return None + flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED + if first: + flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE + h = _winapi.CreateNamedPipe( + self._address, flags, + _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | + _winapi.PIPE_WAIT, + _winapi.PIPE_UNLIMITED_INSTANCES, + windows_utils.BUFSIZE, windows_utils.BUFSIZE, + _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + pipe = windows_utils.PipeHandle(h) + self._free_instances.add(pipe) + return pipe + + def closed(self): + return (self._address is None) + + def close(self): + if self._accept_pipe_future is not None: + self._accept_pipe_future.cancel() + self._accept_pipe_future = None + # Close all instances which have not been connected to by a client. + if self._address is not None: + for pipe in self._free_instances: + pipe.close() + self._pipe = None + self._address = None + self._free_instances.clear() + + __del__ = close + + +class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop): + """Windows version of selector event loop.""" + + def _socketpair(self): + return windows_utils.socketpair() + + +class ProactorEventLoop(proactor_events.BaseProactorEventLoop): + """Windows version of proactor event loop using IOCP.""" + + def __init__(self, proactor=None): + if proactor is None: + proactor = IocpProactor() + super().__init__(proactor) + + def _socketpair(self): + return windows_utils.socketpair() + + @coroutine + def create_pipe_connection(self, protocol_factory, address): + f = self._proactor.connect_pipe(address) + pipe = yield from f + protocol = protocol_factory() + trans = self._make_duplex_pipe_transport(pipe, protocol, + extra={'addr': address}) + return trans, protocol + + @coroutine + def start_serving_pipe(self, protocol_factory, address): + server = PipeServer(address) + + def loop_accept_pipe(f=None): + pipe = None + try: + if f: + pipe = f.result() + server._free_instances.discard(pipe) + + if server.closed(): + # A client connected before the server was closed: + # drop the client (close the pipe) and exit + pipe.close() + return + + protocol = protocol_factory() + self._make_duplex_pipe_transport( + pipe, protocol, extra={'addr': address}) + + pipe = server._get_unconnected_pipe() + if pipe is None: + return + + f = self._proactor.accept_pipe(pipe) + except OSError as exc: + if pipe and pipe.fileno() != -1: + self.call_exception_handler({ + 'message': 'Pipe accept failed', + 'exception': exc, + 'pipe': pipe, + }) + pipe.close() + elif self._debug: + logger.warning("Accept pipe failed on pipe %r", + pipe, exc_info=True) + except futures.CancelledError: + if pipe: + pipe.close() + else: + server._accept_pipe_future = f + f.add_done_callback(loop_accept_pipe) + + self.call_soon(loop_accept_pipe) + return [server] + + @coroutine + def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): + waiter = self.create_future() + transp = _WindowsSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + try: + yield from waiter + except Exception as exc: + # Workaround CPython bug #23353: using yield/yield-from in an + # except block of a generator doesn't clear properly sys.exc_info() + err = exc + else: + err = None + + if err is not None: + transp.close() + yield from transp._wait() + raise err + + return transp + + +class IocpProactor: + """Proactor implementation using IOCP.""" + + def __init__(self, concurrency=0xffffffff): + self._loop = None + self._results = [] + self._iocp = _overlapped.CreateIoCompletionPort( + _overlapped.INVALID_HANDLE_VALUE, NULL, 0, concurrency) + self._cache = {} + self._registered = weakref.WeakSet() + self._unregistered = [] + self._stopped_serving = weakref.WeakSet() + + def __repr__(self): + return ('<%s overlapped#=%s result#=%s>' + % (self.__class__.__name__, len(self._cache), + len(self._results))) + + def set_loop(self, loop): + self._loop = loop + + def select(self, timeout=None): + if not self._results: + self._poll(timeout) + tmp = self._results + self._results = [] + return tmp + + def _result(self, value): + fut = self._loop.create_future() + fut.set_result(value) + return fut + + def recv(self, conn, nbytes, flags=0): + self._register_with_iocp(conn) + ov = _overlapped.Overlapped(NULL) + try: + if isinstance(conn, socket.socket): + ov.WSARecv(conn.fileno(), nbytes, flags) + else: + ov.ReadFile(conn.fileno(), nbytes) + except BrokenPipeError: + return self._result(b'') + + def finish_recv(trans, key, ov): + try: + return ov.getresult() + except OSError as exc: + if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: + raise ConnectionResetError(*exc.args) + else: + raise + + return self._register(ov, conn, finish_recv) + + def send(self, conn, buf, flags=0): + self._register_with_iocp(conn) + ov = _overlapped.Overlapped(NULL) + if isinstance(conn, socket.socket): + ov.WSASend(conn.fileno(), buf, flags) + else: + ov.WriteFile(conn.fileno(), buf) + + def finish_send(trans, key, ov): + try: + return ov.getresult() + except OSError as exc: + if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: + raise ConnectionResetError(*exc.args) + else: + raise + + return self._register(ov, conn, finish_send) + + def accept(self, listener): + self._register_with_iocp(listener) + conn = self._get_accept_socket(listener.family) + ov = _overlapped.Overlapped(NULL) + ov.AcceptEx(listener.fileno(), conn.fileno()) + + def finish_accept(trans, key, ov): + ov.getresult() + # Use SO_UPDATE_ACCEPT_CONTEXT so getsockname() etc work. + buf = struct.pack('@P', listener.fileno()) + conn.setsockopt(socket.SOL_SOCKET, + _overlapped.SO_UPDATE_ACCEPT_CONTEXT, buf) + conn.settimeout(listener.gettimeout()) + return conn, conn.getpeername() + + @coroutine + def accept_coro(future, conn): + # Coroutine closing the accept socket if the future is cancelled + try: + yield from future + except futures.CancelledError: + conn.close() + raise + + future = self._register(ov, listener, finish_accept) + coro = accept_coro(future, conn) + tasks.ensure_future(coro, loop=self._loop) + return future + + def connect(self, conn, address): + self._register_with_iocp(conn) + # The socket needs to be locally bound before we call ConnectEx(). + try: + _overlapped.BindLocal(conn.fileno(), conn.family) + except OSError as e: + if e.winerror != errno.WSAEINVAL: + raise + # Probably already locally bound; check using getsockname(). + if conn.getsockname()[1] == 0: + raise + ov = _overlapped.Overlapped(NULL) + ov.ConnectEx(conn.fileno(), address) + + def finish_connect(trans, key, ov): + ov.getresult() + # Use SO_UPDATE_CONNECT_CONTEXT so getsockname() etc work. + conn.setsockopt(socket.SOL_SOCKET, + _overlapped.SO_UPDATE_CONNECT_CONTEXT, 0) + return conn + + return self._register(ov, conn, finish_connect) + + def accept_pipe(self, pipe): + self._register_with_iocp(pipe) + ov = _overlapped.Overlapped(NULL) + connected = ov.ConnectNamedPipe(pipe.fileno()) + + if connected: + # ConnectNamePipe() failed with ERROR_PIPE_CONNECTED which means + # that the pipe is connected. There is no need to wait for the + # completion of the connection. + return self._result(pipe) + + def finish_accept_pipe(trans, key, ov): + ov.getresult() + return pipe + + return self._register(ov, pipe, finish_accept_pipe) + + @coroutine + def connect_pipe(self, address): + delay = CONNECT_PIPE_INIT_DELAY + while True: + # Unfortunately there is no way to do an overlapped connect to a pipe. + # Call CreateFile() in a loop until it doesn't fail with + # ERROR_PIPE_BUSY + try: + handle = _overlapped.ConnectPipe(address) + break + except OSError as exc: + if exc.winerror != _overlapped.ERROR_PIPE_BUSY: + raise + + # ConnectPipe() failed with ERROR_PIPE_BUSY: retry later + delay = min(delay * 2, CONNECT_PIPE_MAX_DELAY) + yield from tasks.sleep(delay, loop=self._loop) + + return windows_utils.PipeHandle(handle) + + def wait_for_handle(self, handle, timeout=None): + """Wait for a handle. + + Return a Future object. The result of the future is True if the wait + completed, or False if the wait did not complete (on timeout). + """ + return self._wait_for_handle(handle, timeout, False) + + def _wait_cancel(self, event, done_callback): + fut = self._wait_for_handle(event, None, True) + # add_done_callback() cannot be used because the wait may only complete + # in IocpProactor.close(), while the event loop is not running. + fut._done_callback = done_callback + return fut + + def _wait_for_handle(self, handle, timeout, _is_cancel): + if timeout is None: + ms = _winapi.INFINITE + else: + # RegisterWaitForSingleObject() has a resolution of 1 millisecond, + # round away from zero to wait *at least* timeout seconds. + ms = math.ceil(timeout * 1e3) + + # We only create ov so we can use ov.address as a key for the cache. + ov = _overlapped.Overlapped(NULL) + wait_handle = _overlapped.RegisterWaitWithQueue( + handle, self._iocp, ov.address, ms) + if _is_cancel: + f = _WaitCancelFuture(ov, handle, wait_handle, loop=self._loop) + else: + f = _WaitHandleFuture(ov, handle, wait_handle, self, + loop=self._loop) + if f._source_traceback: + del f._source_traceback[-1] + + def finish_wait_for_handle(trans, key, ov): + # Note that this second wait means that we should only use + # this with handles types where a successful wait has no + # effect. So events or processes are all right, but locks + # or semaphores are not. Also note if the handle is + # signalled and then quickly reset, then we may return + # False even though we have not timed out. + return f._poll() + + self._cache[ov.address] = (f, ov, 0, finish_wait_for_handle) + return f + + def _register_with_iocp(self, obj): + # To get notifications of finished ops on this objects sent to the + # completion port, were must register the handle. + if obj not in self._registered: + self._registered.add(obj) + _overlapped.CreateIoCompletionPort(obj.fileno(), self._iocp, 0, 0) + # XXX We could also use SetFileCompletionNotificationModes() + # to avoid sending notifications to completion port of ops + # that succeed immediately. + + def _register(self, ov, obj, callback): + # Return a future which will be set with the result of the + # operation when it completes. The future's value is actually + # the value returned by callback(). + f = _OverlappedFuture(ov, loop=self._loop) + if f._source_traceback: + del f._source_traceback[-1] + if not ov.pending: + # The operation has completed, so no need to postpone the + # work. We cannot take this short cut if we need the + # NumberOfBytes, CompletionKey values returned by + # PostQueuedCompletionStatus(). + try: + value = callback(None, None, ov) + except OSError as e: + f.set_exception(e) + else: + f.set_result(value) + # Even if GetOverlappedResult() was called, we have to wait for the + # notification of the completion in GetQueuedCompletionStatus(). + # Register the overlapped operation to keep a reference to the + # OVERLAPPED object, otherwise the memory is freed and Windows may + # read uninitialized memory. + + # Register the overlapped operation for later. Note that + # we only store obj to prevent it from being garbage + # collected too early. + self._cache[ov.address] = (f, ov, obj, callback) + return f + + def _unregister(self, ov): + """Unregister an overlapped object. + + Call this method when its future has been cancelled. The event can + already be signalled (pending in the proactor event queue). It is also + safe if the event is never signalled (because it was cancelled). + """ + self._unregistered.append(ov) + + def _get_accept_socket(self, family): + s = socket.socket(family) + s.settimeout(0) + return s + + def _poll(self, timeout=None): + if timeout is None: + ms = INFINITE + elif timeout < 0: + raise ValueError("negative timeout") + else: + # GetQueuedCompletionStatus() has a resolution of 1 millisecond, + # round away from zero to wait *at least* timeout seconds. + ms = math.ceil(timeout * 1e3) + if ms >= INFINITE: + raise ValueError("timeout too big") + + while True: + status = _overlapped.GetQueuedCompletionStatus(self._iocp, ms) + if status is None: + break + ms = 0 + + err, transferred, key, address = status + try: + f, ov, obj, callback = self._cache.pop(address) + except KeyError: + if self._loop.get_debug(): + self._loop.call_exception_handler({ + 'message': ('GetQueuedCompletionStatus() returned an ' + 'unexpected event'), + 'status': ('err=%s transferred=%s key=%#x address=%#x' + % (err, transferred, key, address)), + }) + + # key is either zero, or it is used to return a pipe + # handle which should be closed to avoid a leak. + if key not in (0, _overlapped.INVALID_HANDLE_VALUE): + _winapi.CloseHandle(key) + continue + + if obj in self._stopped_serving: + f.cancel() + # Don't call the callback if _register() already read the result or + # if the overlapped has been cancelled + elif not f.done(): + try: + value = callback(transferred, key, ov) + except OSError as e: + f.set_exception(e) + self._results.append(f) + else: + f.set_result(value) + self._results.append(f) + + # Remove unregisted futures + for ov in self._unregistered: + self._cache.pop(ov.address, None) + self._unregistered.clear() + + def _stop_serving(self, obj): + # obj is a socket or pipe handle. It will be closed in + # BaseProactorEventLoop._stop_serving() which will make any + # pending operations fail quickly. + self._stopped_serving.add(obj) + + def close(self): + # Cancel remaining registered operations. + for address, (fut, ov, obj, callback) in list(self._cache.items()): + if fut.cancelled(): + # Nothing to do with cancelled futures + pass + elif isinstance(fut, _WaitCancelFuture): + # _WaitCancelFuture must not be cancelled + pass + else: + try: + fut.cancel() + except OSError as exc: + if self._loop is not None: + context = { + 'message': 'Cancelling a future failed', + 'exception': exc, + 'future': fut, + } + if fut._source_traceback: + context['source_traceback'] = fut._source_traceback + self._loop.call_exception_handler(context) + + while self._cache: + if not self._poll(1): + logger.debug('taking long time to close proactor') + + self._results = [] + if self._iocp is not None: + _winapi.CloseHandle(self._iocp) + self._iocp = None + + def __del__(self): + self.close() + + +class _WindowsSubprocessTransport(base_subprocess.BaseSubprocessTransport): + + def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): + self._proc = windows_utils.Popen( + args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, + bufsize=bufsize, **kwargs) + + def callback(f): + returncode = self._proc.poll() + self._process_exited(returncode) + + f = self._loop._proactor.wait_for_handle(int(self._proc._handle)) + f.add_done_callback(callback) + + +SelectorEventLoop = _WindowsSelectorEventLoop + + +class _WindowsDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): + _loop_factory = SelectorEventLoop + + +DefaultEventLoopPolicy = _WindowsDefaultEventLoopPolicy diff --git a/thirdparty/asyncio/asyncio/windows_utils.py b/thirdparty/asyncio/asyncio/windows_utils.py new file mode 100644 index 0000000..870cd13 --- /dev/null +++ b/thirdparty/asyncio/asyncio/windows_utils.py @@ -0,0 +1,223 @@ +""" +Various Windows specific bits and pieces +""" + +import sys + +if sys.platform != 'win32': # pragma: no cover + raise ImportError('win32 only') + +import _winapi +import itertools +import msvcrt +import os +import socket +import subprocess +import tempfile +import warnings + + +__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle'] + + +# Constants/globals + + +BUFSIZE = 8192 +PIPE = subprocess.PIPE +STDOUT = subprocess.STDOUT +_mmap_counter = itertools.count() + + +if hasattr(socket, 'socketpair'): + # Since Python 3.5, socket.socketpair() is now also available on Windows + socketpair = socket.socketpair +else: + # Replacement for socket.socketpair() + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): + """A socket pair usable as a self-pipe, for Windows. + + Origin: https://gist.github.com/4325783, by Geert Jansen. + Public domain. + """ + if family == socket.AF_INET: + host = '127.0.0.1' + elif family == socket.AF_INET6: + host = '::1' + else: + raise ValueError("Only AF_INET and AF_INET6 socket address " + "families are supported") + if type != socket.SOCK_STREAM: + raise ValueError("Only SOCK_STREAM socket type is supported") + if proto != 0: + raise ValueError("Only protocol zero is supported") + + # We create a connected TCP socket. Note the trick with setblocking(0) + # that prevents us from having to create a thread. + lsock = socket.socket(family, type, proto) + try: + lsock.bind((host, 0)) + lsock.listen(1) + # On IPv6, ignore flow_info and scope_id + addr, port = lsock.getsockname()[:2] + csock = socket.socket(family, type, proto) + try: + csock.setblocking(False) + try: + csock.connect((addr, port)) + except (BlockingIOError, InterruptedError): + pass + csock.setblocking(True) + ssock, _ = lsock.accept() + except: + csock.close() + raise + finally: + lsock.close() + return (ssock, csock) + + +# Replacement for os.pipe() using handles instead of fds + + +def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): + """Like os.pipe() but with overlapped support and using handles not fds.""" + address = tempfile.mktemp(prefix=r'\\.\pipe\python-pipe-%d-%d-' % + (os.getpid(), next(_mmap_counter))) + + if duplex: + openmode = _winapi.PIPE_ACCESS_DUPLEX + access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE + obsize, ibsize = bufsize, bufsize + else: + openmode = _winapi.PIPE_ACCESS_INBOUND + access = _winapi.GENERIC_WRITE + obsize, ibsize = 0, bufsize + + openmode |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE + + if overlapped[0]: + openmode |= _winapi.FILE_FLAG_OVERLAPPED + + if overlapped[1]: + flags_and_attribs = _winapi.FILE_FLAG_OVERLAPPED + else: + flags_and_attribs = 0 + + h1 = h2 = None + try: + h1 = _winapi.CreateNamedPipe( + address, openmode, _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + + h2 = _winapi.CreateFile( + address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, + flags_and_attribs, _winapi.NULL) + + ov = _winapi.ConnectNamedPipe(h1, overlapped=True) + ov.GetOverlappedResult(True) + return h1, h2 + except: + if h1 is not None: + _winapi.CloseHandle(h1) + if h2 is not None: + _winapi.CloseHandle(h2) + raise + + +# Wrapper for a pipe handle + + +class PipeHandle: + """Wrapper for an overlapped pipe handle which is vaguely file-object like. + + The IOCP event loop can use these instead of socket objects. + """ + def __init__(self, handle): + self._handle = handle + + def __repr__(self): + if self._handle is not None: + handle = 'handle=%r' % self._handle + else: + handle = 'closed' + return '<%s %s>' % (self.__class__.__name__, handle) + + @property + def handle(self): + return self._handle + + def fileno(self): + if self._handle is None: + raise ValueError("I/O operatioon on closed pipe") + return self._handle + + def close(self, *, CloseHandle=_winapi.CloseHandle): + if self._handle is not None: + CloseHandle(self._handle) + self._handle = None + + def __del__(self): + if self._handle is not None: + warnings.warn("unclosed %r" % self, ResourceWarning) + self.close() + + def __enter__(self): + return self + + def __exit__(self, t, v, tb): + self.close() + + +# Replacement for subprocess.Popen using overlapped pipe handles + + +class Popen(subprocess.Popen): + """Replacement for subprocess.Popen using overlapped pipe handles. + + The stdin, stdout, stderr are None or instances of PipeHandle. + """ + def __init__(self, args, stdin=None, stdout=None, stderr=None, **kwds): + assert not kwds.get('universal_newlines') + assert kwds.get('bufsize', 0) == 0 + stdin_rfd = stdout_wfd = stderr_wfd = None + stdin_wh = stdout_rh = stderr_rh = None + if stdin == PIPE: + stdin_rh, stdin_wh = pipe(overlapped=(False, True), duplex=True) + stdin_rfd = msvcrt.open_osfhandle(stdin_rh, os.O_RDONLY) + else: + stdin_rfd = stdin + if stdout == PIPE: + stdout_rh, stdout_wh = pipe(overlapped=(True, False)) + stdout_wfd = msvcrt.open_osfhandle(stdout_wh, 0) + else: + stdout_wfd = stdout + if stderr == PIPE: + stderr_rh, stderr_wh = pipe(overlapped=(True, False)) + stderr_wfd = msvcrt.open_osfhandle(stderr_wh, 0) + elif stderr == STDOUT: + stderr_wfd = stdout_wfd + else: + stderr_wfd = stderr + try: + super().__init__(args, stdin=stdin_rfd, stdout=stdout_wfd, + stderr=stderr_wfd, **kwds) + except: + for h in (stdin_wh, stdout_rh, stderr_rh): + if h is not None: + _winapi.CloseHandle(h) + raise + else: + if stdin_wh is not None: + self.stdin = PipeHandle(stdin_wh) + if stdout_rh is not None: + self.stdout = PipeHandle(stdout_rh) + if stderr_rh is not None: + self.stderr = PipeHandle(stderr_rh) + finally: + if stdin == PIPE: + os.close(stdin_rfd) + if stdout == PIPE: + os.close(stdout_wfd) + if stderr == PIPE: + os.close(stderr_wfd) diff --git a/thirdparty/asyncio/check.py b/thirdparty/asyncio/check.py new file mode 100644 index 0000000..6db82d6 --- /dev/null +++ b/thirdparty/asyncio/check.py @@ -0,0 +1,45 @@ +"""Search for lines >= 80 chars or with trailing whitespace.""" + +import os +import sys + + +def main(): + args = sys.argv[1:] or os.curdir + for arg in args: + if os.path.isdir(arg): + for dn, dirs, files in os.walk(arg): + for fn in sorted(files): + if fn.endswith('.py'): + process(os.path.join(dn, fn)) + dirs[:] = [d for d in dirs if d[0] != '.'] + dirs.sort() + else: + process(arg) + + +def isascii(x): + try: + x.encode('ascii') + return True + except UnicodeError: + return False + + +def process(fn): + try: + f = open(fn) + except IOError as err: + print(err) + return + try: + for i, line in enumerate(f): + line = line.rstrip('\n') + sline = line.rstrip() + if len(line) >= 80 or line != sline or not isascii(line): + print('{}:{:d}:{}{}'.format( + fn, i+1, sline, '_' * (len(line) - len(sline)))) + finally: + f.close() + +main() diff --git a/thirdparty/asyncio/examples/cacheclt.py b/thirdparty/asyncio/examples/cacheclt.py new file mode 100644 index 0000000..b11a4d1 --- /dev/null +++ b/thirdparty/asyncio/examples/cacheclt.py @@ -0,0 +1,213 @@ +"""Client for cache server. + +See cachesvr.py for protocol description. +""" + +import argparse +import asyncio +from asyncio import test_utils +import json +import logging + +ARGS = argparse.ArgumentParser(description='Cache client example.') +ARGS.add_argument( + '--tls', action='store_true', dest='tls', + default=False, help='Use TLS') +ARGS.add_argument( + '--iocp', action='store_true', dest='iocp', + default=False, help='Use IOCP event loop (Windows only)') +ARGS.add_argument( + '--host', action='store', dest='host', + default='localhost', help='Host name') +ARGS.add_argument( + '--port', action='store', dest='port', + default=54321, type=int, help='Port number') +ARGS.add_argument( + '--timeout', action='store', dest='timeout', + default=5, type=float, help='Timeout') +ARGS.add_argument( + '--max_backoff', action='store', dest='max_backoff', + default=5, type=float, help='Max backoff on reconnect') +ARGS.add_argument( + '--ntasks', action='store', dest='ntasks', + default=10, type=int, help='Number of tester tasks') +ARGS.add_argument( + '--ntries', action='store', dest='ntries', + default=5, type=int, help='Number of request tries before giving up') + + +args = ARGS.parse_args() + + +class CacheClient: + """Multiplexing cache client. + + This wraps a single connection to the cache client. The + connection is automatically re-opened when an error occurs. + + Multiple tasks may share this object; the requests will be + serialized. + + The public API is get(), set(), delete() (all are coroutines). + """ + + def __init__(self, host, port, sslctx=None, loop=None): + self.host = host + self.port = port + self.sslctx = sslctx + self.loop = loop + self.todo = set() + self.initialized = False + self.task = asyncio.Task(self.activity(), loop=self.loop) + + @asyncio.coroutine + def get(self, key): + resp = yield from self.request('get', key) + if resp is None: + return None + return resp.get('value') + + @asyncio.coroutine + def set(self, key, value): + resp = yield from self.request('set', key, value) + if resp is None: + return False + return resp.get('status') == 'ok' + + @asyncio.coroutine + def delete(self, key): + resp = yield from self.request('delete', key) + if resp is None: + return False + return resp.get('status') == 'ok' + + @asyncio.coroutine + def request(self, type, key, value=None): + assert not self.task.done() + data = {'type': type, 'key': key} + if value is not None: + data['value'] = value + payload = json.dumps(data).encode('utf8') + waiter = asyncio.Future(loop=self.loop) + if self.initialized: + try: + yield from self.send(payload, waiter) + except IOError: + self.todo.add((payload, waiter)) + else: + self.todo.add((payload, waiter)) + return (yield from waiter) + + @asyncio.coroutine + def activity(self): + backoff = 0 + while True: + try: + self.reader, self.writer = yield from asyncio.open_connection( + self.host, self.port, ssl=self.sslctx, loop=self.loop) + except Exception as exc: + backoff = min(args.max_backoff, backoff + (backoff//2) + 1) + logging.info('Error connecting: %r; sleep %s', exc, backoff) + yield from asyncio.sleep(backoff, loop=self.loop) + continue + backoff = 0 + self.next_id = 0 + self.pending = {} + self. initialized = True + try: + while self.todo: + payload, waiter = self.todo.pop() + if not waiter.done(): + yield from self.send(payload, waiter) + while True: + resp_id, resp = yield from self.process() + if resp_id in self.pending: + payload, waiter = self.pending.pop(resp_id) + if not waiter.done(): + waiter.set_result(resp) + except Exception as exc: + self.initialized = False + self.writer.close() + while self.pending: + req_id, pair = self.pending.popitem() + payload, waiter = pair + if not waiter.done(): + self.todo.add(pair) + logging.info('Error processing: %r', exc) + + @asyncio.coroutine + def send(self, payload, waiter): + self.next_id += 1 + req_id = self.next_id + frame = 'request %d %d\n' % (req_id, len(payload)) + self.writer.write(frame.encode('ascii')) + self.writer.write(payload) + self.pending[req_id] = payload, waiter + yield from self.writer.drain() + + @asyncio.coroutine + def process(self): + frame = yield from self.reader.readline() + if not frame: + raise EOFError() + head, tail = frame.split(None, 1) + if head == b'error': + raise IOError('OOB error: %r' % tail) + if head != b'response': + raise IOError('Bad frame: %r' % frame) + resp_id, resp_size = map(int, tail.split()) + data = yield from self.reader.readexactly(resp_size) + if len(data) != resp_size: + raise EOFError() + resp = json.loads(data.decode('utf8')) + return resp_id, resp + + +def main(): + asyncio.set_event_loop(None) + if args.iocp: + from asyncio.windows_events import ProactorEventLoop + loop = ProactorEventLoop() + else: + loop = asyncio.new_event_loop() + sslctx = None + if args.tls: + sslctx = test_utils.dummy_ssl_context() + cache = CacheClient(args.host, args.port, sslctx=sslctx, loop=loop) + try: + loop.run_until_complete( + asyncio.gather( + *[testing(i, cache, loop) for i in range(args.ntasks)], + loop=loop)) + finally: + loop.close() + + +@asyncio.coroutine +def testing(label, cache, loop): + + def w(g): + return asyncio.wait_for(g, args.timeout, loop=loop) + + key = 'foo-%s' % label + while True: + logging.info('%s %s', label, '-'*20) + try: + ret = yield from w(cache.set(key, 'hello-%s-world' % label)) + logging.info('%s set %s', label, ret) + ret = yield from w(cache.get(key)) + logging.info('%s get %s', label, ret) + ret = yield from w(cache.delete(key)) + logging.info('%s del %s', label, ret) + ret = yield from w(cache.get(key)) + logging.info('%s get2 %s', label, ret) + except asyncio.TimeoutError: + logging.warn('%s Timeout', label) + except Exception as exc: + logging.exception('%s Client exception: %r', label, exc) + break + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() diff --git a/thirdparty/asyncio/examples/cachesvr.py b/thirdparty/asyncio/examples/cachesvr.py new file mode 100644 index 0000000..053f9c2 --- /dev/null +++ b/thirdparty/asyncio/examples/cachesvr.py @@ -0,0 +1,249 @@ +"""A simple memcache-like server. + +The basic data structure maintained is a single in-memory dictionary +mapping string keys to string values, with operations get, set and +delete. (Both keys and values may contain Unicode.) + +This is a TCP server listening on port 54321. There is no +authentication. + +Requests provide an operation and return a response. A connection may +be used for multiple requests. The connection is closed when a client +sends a bad request. + +If a client is idle for over 5 seconds (i.e., it does not send another +request, or fails to read the whole response, within this time), it is +disconnected. + +Framing of requests and responses within a connection uses a +line-based protocol. The first line of a request is the frame header +and contains three whitespace-delimited token followed by LF or CRLF: + +- the keyword 'request' +- a decimal request ID; the first request is '1', the second '2', etc. +- a decimal byte count giving the size of the rest of the request + +Note that the requests ID *must* be consecutive and start at '1' for +each connection. + +Response frames look the same except the keyword is 'response'. The +response ID matches the request ID. There should be exactly one +response to each request and responses should be seen in the same +order as the requests. + +After the frame, individual requests and responses are JSON encoded. + +If the frame header or the JSON request body cannot be parsed, an +unframed error message (always starting with 'error') is written back +and the connection is closed. + +JSON-encoded requests can be: + +- {"type": "get", "key": } +- {"type": "set", "key": , "value": } +- {"type": "delete", "key": } + +Responses are also JSON-encoded: + +- {"status": "ok", "value": } # Successful get request +- {"status": "ok"} # Successful set or delete request +- {"status": "notfound"} # Key not found for get or delete request + +If the request is valid JSON but cannot be handled (e.g., the type or +key field is absent or invalid), an error response of the following +form is returned, but the connection is not closed: + +- {"error": } +""" + +import argparse +import asyncio +import json +import logging +import os +import random + +ARGS = argparse.ArgumentParser(description='Cache server example.') +ARGS.add_argument( + '--tls', action='store_true', dest='tls', + default=False, help='Use TLS') +ARGS.add_argument( + '--iocp', action='store_true', dest='iocp', + default=False, help='Use IOCP event loop (Windows only)') +ARGS.add_argument( + '--host', action='store', dest='host', + default='localhost', help='Host name') +ARGS.add_argument( + '--port', action='store', dest='port', + default=54321, type=int, help='Port number') +ARGS.add_argument( + '--timeout', action='store', dest='timeout', + default=5, type=float, help='Timeout') +ARGS.add_argument( + '--random_failure_percent', action='store', dest='fail_percent', + default=0, type=float, help='Fail randomly N percent of the time') +ARGS.add_argument( + '--random_failure_sleep', action='store', dest='fail_sleep', + default=0, type=float, help='Sleep time when randomly failing') +ARGS.add_argument( + '--random_response_sleep', action='store', dest='resp_sleep', + default=0, type=float, help='Sleep time before responding') + +args = ARGS.parse_args() + + +class Cache: + + def __init__(self, loop): + self.loop = loop + self.table = {} + + @asyncio.coroutine + def handle_client(self, reader, writer): + # Wrapper to log stuff and close writer (i.e., transport). + peer = writer.get_extra_info('socket').getpeername() + logging.info('got a connection from %s', peer) + try: + yield from self.frame_parser(reader, writer) + except Exception as exc: + logging.error('error %r from %s', exc, peer) + else: + logging.info('end connection from %s', peer) + finally: + writer.close() + + @asyncio.coroutine + def frame_parser(self, reader, writer): + # This takes care of the framing. + last_request_id = 0 + while True: + # Read the frame header, parse it, read the data. + # NOTE: The readline() and readexactly() calls will hang + # if the client doesn't send enough data but doesn't + # disconnect either. We add a timeout to each. (But the + # timeout should really be implemented by StreamReader.) + framing_b = yield from asyncio.wait_for( + reader.readline(), + timeout=args.timeout, loop=self.loop) + if random.random()*100 < args.fail_percent: + logging.warn('Inserting random failure') + yield from asyncio.sleep(args.fail_sleep*random.random(), + loop=self.loop) + writer.write(b'error random failure\r\n') + break + logging.debug('framing_b = %r', framing_b) + if not framing_b: + break # Clean close. + try: + frame_keyword, request_id_b, byte_count_b = framing_b.split() + except ValueError: + writer.write(b'error unparseable frame\r\n') + break + if frame_keyword != b'request': + writer.write(b'error frame does not start with request\r\n') + break + try: + request_id, byte_count = int(request_id_b), int(byte_count_b) + except ValueError: + writer.write(b'error unparsable frame parameters\r\n') + break + if request_id != last_request_id + 1 or byte_count < 2: + writer.write(b'error invalid frame parameters\r\n') + break + last_request_id = request_id + request_b = yield from asyncio.wait_for( + reader.readexactly(byte_count), + timeout=args.timeout, loop=self.loop) + try: + request = json.loads(request_b.decode('utf8')) + except ValueError: + writer.write(b'error unparsable json\r\n') + break + response = self.handle_request(request) # Not a coroutine. + if response is None: + writer.write(b'error unhandlable request\r\n') + break + response_b = json.dumps(response).encode('utf8') + b'\r\n' + byte_count = len(response_b) + framing_s = 'response {} {}\r\n'.format(request_id, byte_count) + writer.write(framing_s.encode('ascii')) + yield from asyncio.sleep(args.resp_sleep*random.random(), + loop=self.loop) + writer.write(response_b) + + def handle_request(self, request): + # This parses one request and farms it out to a specific handler. + # Return None for all errors. + if not isinstance(request, dict): + return {'error': 'request is not a dict'} + request_type = request.get('type') + if request_type is None: + return {'error': 'no type in request'} + if request_type not in {'get', 'set', 'delete'}: + return {'error': 'unknown request type'} + key = request.get('key') + if not isinstance(key, str): + return {'error': 'key is not a string'} + if request_type == 'get': + return self.handle_get(key) + if request_type == 'set': + value = request.get('value') + if not isinstance(value, str): + return {'error': 'value is not a string'} + return self.handle_set(key, value) + if request_type == 'delete': + return self.handle_delete(key) + assert False, 'bad request type' # Should have been caught above. + + def handle_get(self, key): + value = self.table.get(key) + if value is None: + return {'status': 'notfound'} + else: + return {'status': 'ok', 'value': value} + + def handle_set(self, key, value): + self.table[key] = value + return {'status': 'ok'} + + def handle_delete(self, key): + if key not in self.table: + return {'status': 'notfound'} + else: + del self.table[key] + return {'status': 'ok'} + + +def main(): + asyncio.set_event_loop(None) + if args.iocp: + from asyncio.windows_events import ProactorEventLoop + loop = ProactorEventLoop() + else: + loop = asyncio.new_event_loop() + sslctx = None + if args.tls: + import ssl + # TODO: take cert/key from args as well. + here = os.path.join(os.path.dirname(__file__), '..', 'tests') + sslctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslctx.options |= ssl.OP_NO_SSLv2 + sslctx.load_cert_chain( + certfile=os.path.join(here, 'ssl_cert.pem'), + keyfile=os.path.join(here, 'ssl_key.pem')) + cache = Cache(loop) + task = asyncio.streams.start_server(cache.handle_client, + args.host, args.port, + ssl=sslctx, loop=loop) + svr = loop.run_until_complete(task) + for sock in svr.sockets: + logging.info('socket %s', sock.getsockname()) + try: + loop.run_forever() + finally: + loop.close() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() diff --git a/thirdparty/asyncio/examples/child_process.py b/thirdparty/asyncio/examples/child_process.py new file mode 100644 index 0000000..3fac175 --- /dev/null +++ b/thirdparty/asyncio/examples/child_process.py @@ -0,0 +1,128 @@ +""" +Example of asynchronous interaction with a child python process. + +This example shows how to attach an existing Popen object and use the low level +transport-protocol API. See shell.py and subprocess_shell.py for higher level +examples. +""" + +import os +import sys + +try: + import asyncio +except ImportError: + # asyncio is not installed + sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + import asyncio + +if sys.platform == 'win32': + from asyncio.windows_utils import Popen, PIPE + from asyncio.windows_events import ProactorEventLoop +else: + from subprocess import Popen, PIPE + +# +# Return a write-only transport wrapping a writable pipe +# + +@asyncio.coroutine +def connect_write_pipe(file): + loop = asyncio.get_event_loop() + transport, _ = yield from loop.connect_write_pipe(asyncio.Protocol, file) + return transport + +# +# Wrap a readable pipe in a stream +# + +@asyncio.coroutine +def connect_read_pipe(file): + loop = asyncio.get_event_loop() + stream_reader = asyncio.StreamReader(loop=loop) + def factory(): + return asyncio.StreamReaderProtocol(stream_reader) + transport, _ = yield from loop.connect_read_pipe(factory, file) + return stream_reader, transport + + +# +# Example +# + +@asyncio.coroutine +def main(loop): + # program which prints evaluation of each expression from stdin + code = r'''if 1: + import os + def writeall(fd, buf): + while buf: + n = os.write(fd, buf) + buf = buf[n:] + while True: + s = os.read(0, 1024) + if not s: + break + s = s.decode('ascii') + s = repr(eval(s)) + '\n' + s = s.encode('ascii') + writeall(1, s) + ''' + + # commands to send to input + commands = iter([b"1+1\n", + b"2**16\n", + b"1/3\n", + b"'x'*50", + b"1/0\n"]) + + # start subprocess and wrap stdin, stdout, stderr + p = Popen([sys.executable, '-c', code], + stdin=PIPE, stdout=PIPE, stderr=PIPE) + + stdin = yield from connect_write_pipe(p.stdin) + stdout, stdout_transport = yield from connect_read_pipe(p.stdout) + stderr, stderr_transport = yield from connect_read_pipe(p.stderr) + + # interact with subprocess + name = {stdout:'OUT', stderr:'ERR'} + registered = {asyncio.Task(stderr.readline()): stderr, + asyncio.Task(stdout.readline()): stdout} + while registered: + # write command + cmd = next(commands, None) + if cmd is None: + stdin.close() + else: + print('>>>', cmd.decode('ascii').rstrip()) + stdin.write(cmd) + + # get and print lines from stdout, stderr + timeout = None + while registered: + done, pending = yield from asyncio.wait( + registered, timeout=timeout, + return_when=asyncio.FIRST_COMPLETED) + if not done: + break + for f in done: + stream = registered.pop(f) + res = f.result() + print(name[stream], res.decode('ascii').rstrip()) + if res != b'': + registered[asyncio.Task(stream.readline())] = stream + timeout = 0.0 + + stdout_transport.close() + stderr_transport.close() + +if __name__ == '__main__': + if sys.platform == 'win32': + loop = ProactorEventLoop() + asyncio.set_event_loop(loop) + else: + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(main(loop)) + finally: + loop.close() diff --git a/thirdparty/asyncio/examples/crawl.py b/thirdparty/asyncio/examples/crawl.py new file mode 100644 index 0000000..4cb76d2 --- /dev/null +++ b/thirdparty/asyncio/examples/crawl.py @@ -0,0 +1,864 @@ +#!/usr/bin/env python3.4 + +"""A simple web crawler.""" + +# TODO: +# - More organized logging (with task ID or URL?). +# - Use logging module for Logger. +# - KeyboardInterrupt in HTML parsing may hang or report unretrieved error. +# - Support gzip encoding. +# - Close connection if HTTP/1.0 response. +# - Add timeouts. (E.g. when switching networks, all seems to hang.) +# - Add arguments to specify TLS settings (e.g. cert/key files). +# - Skip reading large non-text/html files? +# - Use ETag and If-Modified-Since? +# - Handle out of file descriptors directly? (How?) + +import argparse +import asyncio +import asyncio.locks +import cgi +from http.client import BadStatusLine +import logging +import re +import sys +import time +import urllib.parse + + +ARGS = argparse.ArgumentParser(description="Web crawler") +ARGS.add_argument( + '--iocp', action='store_true', dest='iocp', + default=False, help='Use IOCP event loop (Windows only)') +ARGS.add_argument( + '--select', action='store_true', dest='select', + default=False, help='Use Select event loop instead of default') +ARGS.add_argument( + 'roots', nargs='*', + default=[], help='Root URL (may be repeated)') +ARGS.add_argument( + '--max_redirect', action='store', type=int, metavar='N', + default=10, help='Limit redirection chains (for 301, 302 etc.)') +ARGS.add_argument( + '--max_tries', action='store', type=int, metavar='N', + default=4, help='Limit retries on network errors') +ARGS.add_argument( + '--max_tasks', action='store', type=int, metavar='N', + default=100, help='Limit concurrent connections') +ARGS.add_argument( + '--max_pool', action='store', type=int, metavar='N', + default=100, help='Limit connection pool size') +ARGS.add_argument( + '--exclude', action='store', metavar='REGEX', + help='Exclude matching URLs') +ARGS.add_argument( + '--strict', action='store_true', + default=True, help='Strict host matching (default)') +ARGS.add_argument( + '--lenient', action='store_false', dest='strict', + default=False, help='Lenient host matching') +ARGS.add_argument( + '-v', '--verbose', action='count', dest='level', + default=1, help='Verbose logging (repeat for more verbose)') +ARGS.add_argument( + '-q', '--quiet', action='store_const', const=0, dest='level', + default=1, help='Quiet logging (opposite of --verbose)') + + +ESCAPES = [('quot', '"'), + ('gt', '>'), + ('lt', '<'), + ('amp', '&') # Must be last. + ] + + +def unescape(url): + """Turn & into &, and so on. + + This is the inverse of cgi.escape(). + """ + for name, char in ESCAPES: + url = url.replace('&' + name + ';', char) + return url + + +def fix_url(url): + """Prefix a schema-less URL with http://.""" + if '://' not in url: + url = 'http://' + url + return url + + +class Logger: + + def __init__(self, level): + self.level = level + + def _log(self, n, args): + if self.level >= n: + print(*args, file=sys.stderr, flush=True) + + def log(self, n, *args): + self._log(n, args) + + def __call__(self, n, *args): + self._log(n, args) + + +class ConnectionPool: + """A connection pool. + + To open a connection, use reserve(). To recycle it, use unreserve(). + + The pool is mostly just a mapping from (host, port, ssl) tuples to + lists of Connections. The currently active connections are *not* + in the data structure; get_connection() takes the connection out, + and recycle_connection() puts it back in. To recycle a + connection, call conn.close(recycle=True). + + There are limits to both the overall pool and the per-key pool. + """ + + def __init__(self, log, max_pool=10, max_tasks=5): + self.log = log + self.max_pool = max_pool # Overall limit. + self.max_tasks = max_tasks # Per-key limit. + self.loop = asyncio.get_event_loop() + self.connections = {} # {(host, port, ssl): [Connection, ...], ...} + self.queue = [] # [Connection, ...] + + def close(self): + """Close all connections available for reuse.""" + for conns in self.connections.values(): + for conn in conns: + conn.close() + self.connections.clear() + self.queue.clear() + + @asyncio.coroutine + def get_connection(self, host, port, ssl): + """Create or reuse a connection.""" + port = port or (443 if ssl else 80) + try: + ipaddrs = yield from self.loop.getaddrinfo(host, port) + except Exception as exc: + self.log(0, 'Exception %r for (%r, %r)' % (exc, host, port)) + raise + self.log(1, '* %s resolves to %s' % + (host, ', '.join(ip[4][0] for ip in ipaddrs))) + + # Look for a reusable connection. + for _, _, _, _, (h, p, *_) in ipaddrs: + key = h, p, ssl + conn = None + conns = self.connections.get(key) + while conns: + conn = conns.pop(0) + self.queue.remove(conn) + if not conns: + del self.connections[key] + if conn.stale(): + self.log(1, 'closing stale connection for', key) + conn.close() # Just in case. + else: + self.log(1, '* Reusing pooled connection', key, + 'FD =', conn.fileno()) + return conn + + # Create a new connection. + conn = Connection(self.log, self, host, port, ssl) + yield from conn.connect() + self.log(1, '* New connection', conn.key, 'FD =', conn.fileno()) + return conn + + def recycle_connection(self, conn): + """Make a connection available for reuse. + + This also prunes the pool if it exceeds the size limits. + """ + if conn.stale(): + conn.close() + return + + key = conn.key + conns = self.connections.setdefault(key, []) + conns.append(conn) + self.queue.append(conn) + + if len(conns) <= self.max_tasks and len(self.queue) <= self.max_pool: + return + + # Prune the queue. + + # Close stale connections for this key first. + stale = [conn for conn in conns if conn.stale()] + if stale: + for conn in stale: + conns.remove(conn) + self.queue.remove(conn) + self.log(1, 'closing stale connection for', key) + conn.close() + if not conns: + del self.connections[key] + + # Close oldest connection(s) for this key if limit reached. + while len(conns) > self.max_tasks: + conn = conns.pop(0) + self.queue.remove(conn) + self.log(1, 'closing oldest connection for', key) + conn.close() + + if len(self.queue) <= self.max_pool: + return + + # Close overall stale connections. + stale = [conn for conn in self.queue if conn.stale()] + if stale: + for conn in stale: + conns = self.connections.get(conn.key) + conns.remove(conn) + self.queue.remove(conn) + self.log(1, 'closing stale connection for', key) + conn.close() + + # Close oldest overall connection(s) if limit reached. + while len(self.queue) > self.max_pool: + conn = self.queue.pop(0) + conns = self.connections.get(conn.key) + c = conns.pop(0) + assert conn == c, (conn.key, conn, c, conns) + self.log(1, 'closing overall oldest connection for', conn.key) + conn.close() + + +class Connection: + + def __init__(self, log, pool, host, port, ssl): + self.log = log + self.pool = pool + self.host = host + self.port = port + self.ssl = ssl + self.reader = None + self.writer = None + self.key = None + + def stale(self): + return self.reader is None or self.reader.at_eof() + + def fileno(self): + writer = self.writer + if writer is not None: + transport = writer.transport + if transport is not None: + sock = transport.get_extra_info('socket') + if sock is not None: + return sock.fileno() + return None + + @asyncio.coroutine + def connect(self): + self.reader, self.writer = yield from asyncio.open_connection( + self.host, self.port, ssl=self.ssl) + peername = self.writer.get_extra_info('peername') + if peername: + self.host, self.port = peername[:2] + else: + self.log(1, 'NO PEERNAME???', self.host, self.port, self.ssl) + self.key = self.host, self.port, self.ssl + + def close(self, recycle=False): + if recycle and not self.stale(): + self.pool.recycle_connection(self) + else: + self.writer.close() + self.pool = self.reader = self.writer = None + + +class Request: + """HTTP request. + + Use connect() to open a connection; send_request() to send the + request; get_response() to receive the response headers. + """ + + def __init__(self, log, url, pool): + self.log = log + self.url = url + self.pool = pool + self.parts = urllib.parse.urlparse(self.url) + self.scheme = self.parts.scheme + assert self.scheme in ('http', 'https'), repr(url) + self.ssl = self.parts.scheme == 'https' + self.netloc = self.parts.netloc + self.hostname = self.parts.hostname + self.port = self.parts.port or (443 if self.ssl else 80) + self.path = (self.parts.path or '/') + self.query = self.parts.query + if self.query: + self.full_path = '%s?%s' % (self.path, self.query) + else: + self.full_path = self.path + self.http_version = 'HTTP/1.1' + self.method = 'GET' + self.headers = [] + self.conn = None + + @asyncio.coroutine + def connect(self): + """Open a connection to the server.""" + self.log(1, '* Connecting to %s:%s using %s for %s' % + (self.hostname, self.port, + 'ssl' if self.ssl else 'tcp', + self.url)) + self.conn = yield from self.pool.get_connection(self.hostname, + self.port, self.ssl) + + def close(self, recycle=False): + """Close the connection, recycle if requested.""" + if self.conn is not None: + if not recycle: + self.log(1, 'closing connection for', self.conn.key) + self.conn.close(recycle) + self.conn = None + + @asyncio.coroutine + def putline(self, line): + """Write a line to the connection. + + Used for the request line and headers. + """ + self.log(2, '>', line) + self.conn.writer.write(line.encode('latin-1') + b'\r\n') + + @asyncio.coroutine + def send_request(self): + """Send the request.""" + request_line = '%s %s %s' % (self.method, self.full_path, + self.http_version) + yield from self.putline(request_line) + # TODO: What if a header is already set? + self.headers.append(('User-Agent', 'asyncio-example-crawl/0.0')) + self.headers.append(('Host', self.netloc)) + self.headers.append(('Accept', '*/*')) + ##self.headers.append(('Accept-Encoding', 'gzip')) + for key, value in self.headers: + line = '%s: %s' % (key, value) + yield from self.putline(line) + yield from self.putline('') + + @asyncio.coroutine + def get_response(self): + """Receive the response.""" + response = Response(self.log, self.conn.reader) + yield from response.read_headers() + return response + + +class Response: + """HTTP response. + + Call read_headers() to receive the request headers. Then check + the status attribute and call get_header() to inspect the headers. + Finally call read() to receive the body. + """ + + def __init__(self, log, reader): + self.log = log + self.reader = reader + self.http_version = None # 'HTTP/1.1' + self.status = None # 200 + self.reason = None # 'Ok' + self.headers = [] # [('Content-Type', 'text/html')] + + @asyncio.coroutine + def getline(self): + """Read one line from the connection.""" + line = (yield from self.reader.readline()).decode('latin-1').rstrip() + self.log(2, '<', line) + return line + + @asyncio.coroutine + def read_headers(self): + """Read the response status and the request headers.""" + status_line = yield from self.getline() + status_parts = status_line.split(None, 2) + if len(status_parts) != 3: + self.log(0, 'bad status_line', repr(status_line)) + raise BadStatusLine(status_line) + self.http_version, status, self.reason = status_parts + self.status = int(status) + while True: + header_line = yield from self.getline() + if not header_line: + break + # TODO: Continuation lines. + key, value = header_line.split(':', 1) + self.headers.append((key, value.strip())) + + def get_redirect_url(self, default=''): + """Inspect the status and return the redirect url if appropriate.""" + if self.status not in (300, 301, 302, 303, 307): + return default + return self.get_header('Location', default) + + def get_header(self, key, default=''): + """Get one header value, using a case insensitive header name.""" + key = key.lower() + for k, v in self.headers: + if k.lower() == key: + return v + return default + + @asyncio.coroutine + def read(self): + """Read the response body. + + This honors Content-Length and Transfer-Encoding: chunked. + """ + nbytes = None + for key, value in self.headers: + if key.lower() == 'content-length': + nbytes = int(value) + break + if nbytes is None: + if self.get_header('transfer-encoding').lower() == 'chunked': + self.log(2, 'parsing chunked response') + blocks = [] + while True: + size_header = yield from self.reader.readline() + if not size_header: + self.log(0, 'premature end of chunked response') + break + self.log(3, 'size_header =', repr(size_header)) + parts = size_header.split(b';') + size = int(parts[0], 16) + if size: + self.log(3, 'reading chunk of', size, 'bytes') + block = yield from self.reader.readexactly(size) + assert len(block) == size, (len(block), size) + blocks.append(block) + crlf = yield from self.reader.readline() + assert crlf == b'\r\n', repr(crlf) + if not size: + break + body = b''.join(blocks) + self.log(1, 'chunked response had', len(body), + 'bytes in', len(blocks), 'blocks') + else: + self.log(3, 'reading until EOF') + body = yield from self.reader.read() + # TODO: Should make sure not to recycle the connection + # in this case. + else: + body = yield from self.reader.readexactly(nbytes) + return body + + +class Fetcher: + """Logic and state for one URL. + + When found in crawler.busy, this represents a URL to be fetched or + in the process of being fetched; when found in crawler.done, this + holds the results from fetching it. + + This is usually associated with a task. This references the + crawler for the connection pool and to add more URLs to its todo + list. + + Call fetch() to do the fetching, then report() to print the results. + """ + + def __init__(self, log, url, crawler, max_redirect=10, max_tries=4): + self.log = log + self.url = url + self.crawler = crawler + # We don't loop resolving redirects here -- we just use this + # to decide whether to add the redirect URL to crawler.todo. + self.max_redirect = max_redirect + # But we do loop to retry on errors a few times. + self.max_tries = max_tries + # Everything we collect from the response goes here. + self.task = None + self.exceptions = [] + self.tries = 0 + self.request = None + self.response = None + self.body = None + self.next_url = None + self.ctype = None + self.pdict = None + self.encoding = None + self.urls = None + self.new_urls = None + + @asyncio.coroutine + def fetch(self): + """Attempt to fetch the contents of the URL. + + If successful, and the data is HTML, extract further links and + add them to the crawler. Redirects are also added back there. + """ + while self.tries < self.max_tries: + self.tries += 1 + self.request = None + try: + self.request = Request(self.log, self.url, self.crawler.pool) + yield from self.request.connect() + yield from self.request.send_request() + self.response = yield from self.request.get_response() + self.body = yield from self.response.read() + h_conn = self.response.get_header('connection').lower() + if h_conn != 'close': + self.request.close(recycle=True) + self.request = None + if self.tries > 1: + self.log(1, 'try', self.tries, 'for', self.url, 'success') + break + except (BadStatusLine, OSError) as exc: + self.exceptions.append(exc) + self.log(1, 'try', self.tries, 'for', self.url, + 'raised', repr(exc)) + ##import pdb; pdb.set_trace() + # Don't reuse the connection in this case. + finally: + if self.request is not None: + self.request.close() + else: + # We never broke out of the while loop, i.e. all tries failed. + self.log(0, 'no success for', self.url, + 'in', self.max_tries, 'tries') + return + next_url = self.response.get_redirect_url() + if next_url: + self.next_url = urllib.parse.urljoin(self.url, next_url) + if self.max_redirect > 0: + self.log(1, 'redirect to', self.next_url, 'from', self.url) + self.crawler.add_url(self.next_url, self.max_redirect-1) + else: + self.log(0, 'redirect limit reached for', self.next_url, + 'from', self.url) + else: + if self.response.status == 200: + self.ctype = self.response.get_header('content-type') + self.pdict = {} + if self.ctype: + self.ctype, self.pdict = cgi.parse_header(self.ctype) + self.encoding = self.pdict.get('charset', 'utf-8') + if self.ctype == 'text/html': + body = self.body.decode(self.encoding, 'replace') + # Replace href with (?:href|src) to follow image links. + self.urls = set(re.findall(r'(?i)href=["\']?([^\s"\'<>]+)', + body)) + if self.urls: + self.log(1, 'got', len(self.urls), + 'distinct urls from', self.url) + self.new_urls = set() + for url in self.urls: + url = unescape(url) + url = urllib.parse.urljoin(self.url, url) + url, frag = urllib.parse.urldefrag(url) + if self.crawler.add_url(url): + self.new_urls.add(url) + + def report(self, stats, file=None): + """Print a report on the state for this URL. + + Also update the Stats instance. + """ + if self.task is not None: + if not self.task.done(): + stats.add('pending') + print(self.url, 'pending', file=file) + return + elif self.task.cancelled(): + stats.add('cancelled') + print(self.url, 'cancelled', file=file) + return + elif self.task.exception(): + stats.add('exception') + exc = self.task.exception() + stats.add('exception_' + exc.__class__.__name__) + print(self.url, exc, file=file) + return + if len(self.exceptions) == self.tries: + stats.add('fail') + exc = self.exceptions[-1] + stats.add('fail_' + str(exc.__class__.__name__)) + print(self.url, 'error', exc, file=file) + elif self.next_url: + stats.add('redirect') + print(self.url, self.response.status, 'redirect', self.next_url, + file=file) + elif self.ctype == 'text/html': + stats.add('html') + size = len(self.body or b'') + stats.add('html_bytes', size) + if self.log.level: + print(self.url, self.response.status, + self.ctype, self.encoding, + size, + '%d/%d' % (len(self.new_urls or ()), len(self.urls or ())), + file=file) + elif self.response is None: + print(self.url, 'no response object') + else: + size = len(self.body or b'') + if self.response.status == 200: + stats.add('other') + stats.add('other_bytes', size) + else: + stats.add('error') + stats.add('error_bytes', size) + stats.add('status_%s' % self.response.status) + print(self.url, self.response.status, + self.ctype, self.encoding, + size, + file=file) + + +class Stats: + """Record stats of various sorts.""" + + def __init__(self): + self.stats = {} + + def add(self, key, count=1): + self.stats[key] = self.stats.get(key, 0) + count + + def report(self, file=None): + for key, count in sorted(self.stats.items()): + print('%10d' % count, key, file=file) + + +class Crawler: + """Crawl a set of URLs. + + This manages three disjoint sets of URLs (todo, busy, done). The + data structures actually store dicts -- the values in todo give + the redirect limit, while the values in busy and done are Fetcher + instances. + """ + def __init__(self, log, + roots, exclude=None, strict=True, # What to crawl. + max_redirect=10, max_tries=4, # Per-url limits. + max_tasks=10, max_pool=10, # Global limits. + ): + self.log = log + self.roots = roots + self.exclude = exclude + self.strict = strict + self.max_redirect = max_redirect + self.max_tries = max_tries + self.max_tasks = max_tasks + self.max_pool = max_pool + self.todo = {} + self.busy = {} + self.done = {} + self.pool = ConnectionPool(self.log, max_pool, max_tasks) + self.root_domains = set() + for root in roots: + parts = urllib.parse.urlparse(root) + host, port = urllib.parse.splitport(parts.netloc) + if not host: + continue + if re.match(r'\A[\d\.]*\Z', host): + self.root_domains.add(host) + else: + host = host.lower() + if self.strict: + self.root_domains.add(host) + if host.startswith('www.'): + self.root_domains.add(host[4:]) + else: + self.root_domains.add('www.' + host) + else: + parts = host.split('.') + if len(parts) > 2: + host = '.'.join(parts[-2:]) + self.root_domains.add(host) + for root in roots: + self.add_url(root) + self.governor = asyncio.locks.Semaphore(max_tasks) + self.termination = asyncio.locks.Condition() + self.t0 = time.time() + self.t1 = None + + def close(self): + """Close resources (currently only the pool).""" + self.pool.close() + + def host_okay(self, host): + """Check if a host should be crawled. + + A literal match (after lowercasing) is always good. For hosts + that don't look like IP addresses, some approximate matches + are okay depending on the strict flag. + """ + host = host.lower() + if host in self.root_domains: + return True + if re.match(r'\A[\d\.]*\Z', host): + return False + if self.strict: + return self._host_okay_strictish(host) + else: + return self._host_okay_lenient(host) + + def _host_okay_strictish(self, host): + """Check if a host should be crawled, strict-ish version. + + This checks for equality modulo an initial 'www.' component. + """ + if host.startswith('www.'): + if host[4:] in self.root_domains: + return True + else: + if 'www.' + host in self.root_domains: + return True + return False + + def _host_okay_lenient(self, host): + """Check if a host should be crawled, lenient version. + + This compares the last two components of the host. + """ + parts = host.split('.') + if len(parts) > 2: + host = '.'.join(parts[-2:]) + return host in self.root_domains + + def add_url(self, url, max_redirect=None): + """Add a URL to the todo list if not seen before.""" + if self.exclude and re.search(self.exclude, url): + return False + parts = urllib.parse.urlparse(url) + if parts.scheme not in ('http', 'https'): + self.log(2, 'skipping non-http scheme in', url) + return False + host, port = urllib.parse.splitport(parts.netloc) + if not self.host_okay(host): + self.log(2, 'skipping non-root host in', url) + return False + if max_redirect is None: + max_redirect = self.max_redirect + if url in self.todo or url in self.busy or url in self.done: + return False + self.log(1, 'adding', url, max_redirect) + self.todo[url] = max_redirect + return True + + @asyncio.coroutine + def crawl(self): + """Run the crawler until all finished.""" + with (yield from self.termination): + while self.todo or self.busy: + if self.todo: + url, max_redirect = self.todo.popitem() + fetcher = Fetcher(self.log, url, + crawler=self, + max_redirect=max_redirect, + max_tries=self.max_tries, + ) + self.busy[url] = fetcher + fetcher.task = asyncio.Task(self.fetch(fetcher)) + else: + yield from self.termination.wait() + self.t1 = time.time() + + @asyncio.coroutine + def fetch(self, fetcher): + """Call the Fetcher's fetch(), with a limit on concurrency. + + Once this returns, move the fetcher from busy to done. + """ + url = fetcher.url + with (yield from self.governor): + try: + yield from fetcher.fetch() # Fetcher gonna fetch. + finally: + # Force GC of the task, so the error is logged. + fetcher.task = None + with (yield from self.termination): + self.done[url] = fetcher + del self.busy[url] + self.termination.notify() + + def report(self, file=None): + """Print a report on all completed URLs.""" + if self.t1 is None: + self.t1 = time.time() + dt = self.t1 - self.t0 + if dt and self.max_tasks: + speed = len(self.done) / dt / self.max_tasks + else: + speed = 0 + stats = Stats() + print('*** Report ***', file=file) + try: + show = [] + show.extend(self.done.items()) + show.extend(self.busy.items()) + show.sort() + for url, fetcher in show: + fetcher.report(stats, file=file) + except KeyboardInterrupt: + print('\nInterrupted', file=file) + print('Finished', len(self.done), + 'urls in %.3f secs' % dt, + '(max_tasks=%d)' % self.max_tasks, + '(%.3f urls/sec/task)' % speed, + file=file) + stats.report(file=file) + print('Todo:', len(self.todo), file=file) + print('Busy:', len(self.busy), file=file) + print('Done:', len(self.done), file=file) + print('Date:', time.ctime(), 'local time', file=file) + + +def main(): + """Main program. + + Parse arguments, set up event loop, run crawler, print report. + """ + args = ARGS.parse_args() + if not args.roots: + print('Use --help for command line help') + return + + log = Logger(args.level) + + if args.iocp: + from asyncio.windows_events import ProactorEventLoop + loop = ProactorEventLoop() + asyncio.set_event_loop(loop) + elif args.select: + loop = asyncio.SelectorEventLoop() + asyncio.set_event_loop(loop) + else: + loop = asyncio.get_event_loop() + + roots = {fix_url(root) for root in args.roots} + + crawler = Crawler(log, + roots, exclude=args.exclude, + strict=args.strict, + max_redirect=args.max_redirect, + max_tries=args.max_tries, + max_tasks=args.max_tasks, + max_pool=args.max_pool, + ) + try: + loop.run_until_complete(crawler.crawl()) # Crawler gonna crawl. + except KeyboardInterrupt: + sys.stderr.flush() + print('\nInterrupted\n') + finally: + crawler.report() + crawler.close() + loop.close() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() diff --git a/thirdparty/asyncio/examples/echo_client_tulip.py b/thirdparty/asyncio/examples/echo_client_tulip.py new file mode 100644 index 0000000..88124ef --- /dev/null +++ b/thirdparty/asyncio/examples/echo_client_tulip.py @@ -0,0 +1,20 @@ +import asyncio + +END = b'Bye-bye!\n' + +@asyncio.coroutine +def echo_client(): + reader, writer = yield from asyncio.open_connection('localhost', 8000) + writer.write(b'Hello, world\n') + writer.write(b'What a fine day it is.\n') + writer.write(END) + while True: + line = yield from reader.readline() + print('received:', line) + if line == END or not line: + break + writer.close() + +loop = asyncio.get_event_loop() +loop.run_until_complete(echo_client()) +loop.close() diff --git a/thirdparty/asyncio/examples/echo_server_tulip.py b/thirdparty/asyncio/examples/echo_server_tulip.py new file mode 100644 index 0000000..8167e54 --- /dev/null +++ b/thirdparty/asyncio/examples/echo_server_tulip.py @@ -0,0 +1,20 @@ +import asyncio + +@asyncio.coroutine +def echo_server(): + yield from asyncio.start_server(handle_connection, 'localhost', 8000) + +@asyncio.coroutine +def handle_connection(reader, writer): + while True: + data = yield from reader.read(8192) + if not data: + break + writer.write(data) + +loop = asyncio.get_event_loop() +loop.run_until_complete(echo_server()) +try: + loop.run_forever() +finally: + loop.close() diff --git a/thirdparty/asyncio/examples/fetch0.py b/thirdparty/asyncio/examples/fetch0.py new file mode 100644 index 0000000..180fcf2 --- /dev/null +++ b/thirdparty/asyncio/examples/fetch0.py @@ -0,0 +1,35 @@ +"""Simplest possible HTTP client.""" + +import sys + +from asyncio import * + + +@coroutine +def fetch(): + r, w = yield from open_connection('python.org', 80) + request = 'GET / HTTP/1.0\r\n\r\n' + print('>', request, file=sys.stderr) + w.write(request.encode('latin-1')) + while True: + line = yield from r.readline() + line = line.decode('latin-1').rstrip() + if not line: + break + print('<', line, file=sys.stderr) + print(file=sys.stderr) + body = yield from r.read() + return body + + +def main(): + loop = get_event_loop() + try: + body = loop.run_until_complete(fetch()) + finally: + loop.close() + print(body.decode('latin-1'), end='') + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/fetch1.py b/thirdparty/asyncio/examples/fetch1.py new file mode 100644 index 0000000..8dbb6e4 --- /dev/null +++ b/thirdparty/asyncio/examples/fetch1.py @@ -0,0 +1,78 @@ +"""Fetch one URL and write its content to stdout. + +This version adds URL parsing (including SSL) and a Response object. +""" + +import sys +import urllib.parse + +from asyncio import * + + +class Response: + + def __init__(self, verbose=True): + self.verbose = verbose + self.http_version = None # 'HTTP/1.1' + self.status = None # 200 + self.reason = None # 'Ok' + self.headers = [] # [('Content-Type', 'text/html')] + + @coroutine + def read(self, reader): + @coroutine + def getline(): + return (yield from reader.readline()).decode('latin-1').rstrip() + status_line = yield from getline() + if self.verbose: print('<', status_line, file=sys.stderr) + self.http_version, status, self.reason = status_line.split(None, 2) + self.status = int(status) + while True: + header_line = yield from getline() + if not header_line: + break + if self.verbose: print('<', header_line, file=sys.stderr) + # TODO: Continuation lines. + key, value = header_line.split(':', 1) + self.headers.append((key, value.strip())) + if self.verbose: print(file=sys.stderr) + + +@coroutine +def fetch(url, verbose=True): + parts = urllib.parse.urlparse(url) + if parts.scheme == 'http': + ssl = False + elif parts.scheme == 'https': + ssl = True + else: + print('URL must use http or https.') + sys.exit(1) + port = parts.port + if port is None: + port = 443 if ssl else 80 + path = parts.path or '/' + if parts.query: + path += '?' + parts.query + request = 'GET %s HTTP/1.0\r\n\r\n' % path + if verbose: + print('>', request, file=sys.stderr, end='') + r, w = yield from open_connection(parts.hostname, port, ssl=ssl) + w.write(request.encode('latin-1')) + response = Response(verbose) + yield from response.read(r) + body = yield from r.read() + return body + + +def main(): + loop = get_event_loop() + try: + body = loop.run_until_complete(fetch(sys.argv[1], '-v' in sys.argv)) + finally: + loop.close() + print(body.decode('latin-1'), end='') + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/fetch2.py b/thirdparty/asyncio/examples/fetch2.py new file mode 100644 index 0000000..7617b59 --- /dev/null +++ b/thirdparty/asyncio/examples/fetch2.py @@ -0,0 +1,141 @@ +"""Fetch one URL and write its content to stdout. + +This version adds a Request object. +""" + +import sys +import urllib.parse +from http.client import BadStatusLine + +from asyncio import * + + +class Request: + + def __init__(self, url, verbose=True): + self.url = url + self.verbose = verbose + self.parts = urllib.parse.urlparse(self.url) + self.scheme = self.parts.scheme + assert self.scheme in ('http', 'https'), repr(url) + self.ssl = self.parts.scheme == 'https' + self.netloc = self.parts.netloc + self.hostname = self.parts.hostname + self.port = self.parts.port or (443 if self.ssl else 80) + self.path = (self.parts.path or '/') + self.query = self.parts.query + if self.query: + self.full_path = '%s?%s' % (self.path, self.query) + else: + self.full_path = self.path + self.http_version = 'HTTP/1.1' + self.method = 'GET' + self.headers = [] + self.reader = None + self.writer = None + + @coroutine + def connect(self): + if self.verbose: + print('* Connecting to %s:%s using %s' % + (self.hostname, self.port, 'ssl' if self.ssl else 'tcp'), + file=sys.stderr) + self.reader, self.writer = yield from open_connection(self.hostname, + self.port, + ssl=self.ssl) + if self.verbose: + print('* Connected to %s' % + (self.writer.get_extra_info('peername'),), + file=sys.stderr) + + def putline(self, line): + self.writer.write(line.encode('latin-1') + b'\r\n') + + @coroutine + def send_request(self): + request = '%s %s %s' % (self.method, self.full_path, self.http_version) + if self.verbose: print('>', request, file=sys.stderr) + self.putline(request) + if 'host' not in {key.lower() for key, _ in self.headers}: + self.headers.insert(0, ('Host', self.netloc)) + for key, value in self.headers: + line = '%s: %s' % (key, value) + if self.verbose: print('>', line, file=sys.stderr) + self.putline(line) + self.putline('') + + @coroutine + def get_response(self): + response = Response(self.reader, self.verbose) + yield from response.read_headers() + return response + + +class Response: + + def __init__(self, reader, verbose=True): + self.reader = reader + self.verbose = verbose + self.http_version = None # 'HTTP/1.1' + self.status = None # 200 + self.reason = None # 'Ok' + self.headers = [] # [('Content-Type', 'text/html')] + + @coroutine + def getline(self): + return (yield from self.reader.readline()).decode('latin-1').rstrip() + + @coroutine + def read_headers(self): + status_line = yield from self.getline() + if self.verbose: print('<', status_line, file=sys.stderr) + status_parts = status_line.split(None, 2) + if len(status_parts) != 3: + raise BadStatusLine(status_line) + self.http_version, status, self.reason = status_parts + self.status = int(status) + while True: + header_line = yield from self.getline() + if not header_line: + break + if self.verbose: print('<', header_line, file=sys.stderr) + # TODO: Continuation lines. + key, value = header_line.split(':', 1) + self.headers.append((key, value.strip())) + if self.verbose: print(file=sys.stderr) + + @coroutine + def read(self): + nbytes = None + for key, value in self.headers: + if key.lower() == 'content-length': + nbytes = int(value) + break + if nbytes is None: + body = yield from self.reader.read() + else: + body = yield from self.reader.readexactly(nbytes) + return body + + +@coroutine +def fetch(url, verbose=True): + request = Request(url, verbose) + yield from request.connect() + yield from request.send_request() + response = yield from request.get_response() + body = yield from response.read() + return body + + +def main(): + loop = get_event_loop() + try: + body = loop.run_until_complete(fetch(sys.argv[1], '-v' in sys.argv)) + finally: + loop.close() + sys.stdout.buffer.write(body) + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/fetch3.py b/thirdparty/asyncio/examples/fetch3.py new file mode 100644 index 0000000..9419afd --- /dev/null +++ b/thirdparty/asyncio/examples/fetch3.py @@ -0,0 +1,230 @@ +"""Fetch one URL and write its content to stdout. + +This version adds a primitive connection pool, redirect following and +chunked transfer-encoding. It also supports a --iocp flag. +""" + +import sys +import urllib.parse +from http.client import BadStatusLine + +from asyncio import * + + +class ConnectionPool: + # TODO: Locking? Close idle connections? + + def __init__(self, verbose=False): + self.verbose = verbose + self.connections = {} # {(host, port, ssl): (reader, writer)} + + def close(self): + for _, writer in self.connections.values(): + writer.close() + + @coroutine + def open_connection(self, host, port, ssl): + port = port or (443 if ssl else 80) + ipaddrs = yield from get_event_loop().getaddrinfo(host, port) + if self.verbose: + print('* %s resolves to %s' % + (host, ', '.join(ip[4][0] for ip in ipaddrs)), + file=sys.stderr) + for _, _, _, _, (h, p, *_) in ipaddrs: + key = h, p, ssl + conn = self.connections.get(key) + if conn: + reader, writer = conn + if reader._eof: + self.connections.pop(key) + continue + if self.verbose: + print('* Reusing pooled connection', key, file=sys.stderr) + return conn + reader, writer = yield from open_connection(host, port, ssl=ssl) + host, port, *_ = writer.get_extra_info('peername') + key = host, port, ssl + self.connections[key] = reader, writer + if self.verbose: + print('* New connection', key, file=sys.stderr) + return reader, writer + + +class Request: + + def __init__(self, url, verbose=True): + self.url = url + self.verbose = verbose + self.parts = urllib.parse.urlparse(self.url) + self.scheme = self.parts.scheme + assert self.scheme in ('http', 'https'), repr(url) + self.ssl = self.parts.scheme == 'https' + self.netloc = self.parts.netloc + self.hostname = self.parts.hostname + self.port = self.parts.port or (443 if self.ssl else 80) + self.path = (self.parts.path or '/') + self.query = self.parts.query + if self.query: + self.full_path = '%s?%s' % (self.path, self.query) + else: + self.full_path = self.path + self.http_version = 'HTTP/1.1' + self.method = 'GET' + self.headers = [] + self.reader = None + self.writer = None + + def vprint(self, *args): + if self.verbose: + print(*args, file=sys.stderr) + + @coroutine + def connect(self, pool): + self.vprint('* Connecting to %s:%s using %s' % + (self.hostname, self.port, 'ssl' if self.ssl else 'tcp')) + self.reader, self.writer = \ + yield from pool.open_connection(self.hostname, + self.port, + ssl=self.ssl) + self.vprint('* Connected to %s' % + (self.writer.get_extra_info('peername'),)) + + @coroutine + def putline(self, line): + self.vprint('>', line) + self.writer.write(line.encode('latin-1') + b'\r\n') + ##yield from self.writer.drain() + + @coroutine + def send_request(self): + request = '%s %s %s' % (self.method, self.full_path, self.http_version) + yield from self.putline(request) + if 'host' not in {key.lower() for key, _ in self.headers}: + self.headers.insert(0, ('Host', self.netloc)) + for key, value in self.headers: + line = '%s: %s' % (key, value) + yield from self.putline(line) + yield from self.putline('') + + @coroutine + def get_response(self): + response = Response(self.reader, self.verbose) + yield from response.read_headers() + return response + + +class Response: + + def __init__(self, reader, verbose=True): + self.reader = reader + self.verbose = verbose + self.http_version = None # 'HTTP/1.1' + self.status = None # 200 + self.reason = None # 'Ok' + self.headers = [] # [('Content-Type', 'text/html')] + + def vprint(self, *args): + if self.verbose: + print(*args, file=sys.stderr) + + @coroutine + def getline(self): + line = (yield from self.reader.readline()).decode('latin-1').rstrip() + self.vprint('<', line) + return line + + @coroutine + def read_headers(self): + status_line = yield from self.getline() + status_parts = status_line.split(None, 2) + if len(status_parts) != 3: + raise BadStatusLine(status_line) + self.http_version, status, self.reason = status_parts + self.status = int(status) + while True: + header_line = yield from self.getline() + if not header_line: + break + # TODO: Continuation lines. + key, value = header_line.split(':', 1) + self.headers.append((key, value.strip())) + + def get_redirect_url(self, default=None): + if self.status not in (300, 301, 302, 303, 307): + return default + return self.get_header('Location', default) + + def get_header(self, key, default=None): + key = key.lower() + for k, v in self.headers: + if k.lower() == key: + return v + return default + + @coroutine + def read(self): + nbytes = None + for key, value in self.headers: + if key.lower() == 'content-length': + nbytes = int(value) + break + if nbytes is None: + if self.get_header('transfer-encoding', '').lower() == 'chunked': + blocks = [] + size = -1 + while size: + size_header = yield from self.reader.readline() + if not size_header: + break + parts = size_header.split(b';') + size = int(parts[0], 16) + if size: + block = yield from self.reader.readexactly(size) + assert len(block) == size, (len(block), size) + blocks.append(block) + crlf = yield from self.reader.readline() + assert crlf == b'\r\n', repr(crlf) + body = b''.join(blocks) + else: + body = yield from self.reader.read() + else: + body = yield from self.reader.readexactly(nbytes) + return body + + +@coroutine +def fetch(url, verbose=True, max_redirect=10): + pool = ConnectionPool(verbose) + try: + for _ in range(max_redirect): + request = Request(url, verbose) + yield from request.connect(pool) + yield from request.send_request() + response = yield from request.get_response() + body = yield from response.read() + next_url = response.get_redirect_url() + if not next_url: + break + url = urllib.parse.urljoin(url, next_url) + print('redirect to', url, file=sys.stderr) + return body + finally: + pool.close() + + +def main(): + if '--iocp' in sys.argv: + from asyncio.windows_events import ProactorEventLoop + loop = ProactorEventLoop() + set_event_loop(loop) + else: + loop = get_event_loop() + try: + body = loop.run_until_complete(fetch(sys.argv[1], '-v' in sys.argv)) + finally: + loop.close() + sys.stdout.buffer.write(body) + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/fuzz_as_completed.py b/thirdparty/asyncio/examples/fuzz_as_completed.py new file mode 100644 index 0000000..123fbf1 --- /dev/null +++ b/thirdparty/asyncio/examples/fuzz_as_completed.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +"""Fuzz tester for as_completed(), by Glenn Langford.""" + +import asyncio +import itertools +import random +import sys + +@asyncio.coroutine +def sleeper(time): + yield from asyncio.sleep(time) + return time + +@asyncio.coroutine +def watcher(tasks,delay=False): + res = [] + for t in asyncio.as_completed(tasks): + r = yield from t + res.append(r) + if delay: + # simulate processing delay + process_time = random.random() / 10 + yield from asyncio.sleep(process_time) + #print(res) + #assert(sorted(res) == res) + if sorted(res) != res: + print('FAIL', res) + print('------------') + else: + print('.', end='') + sys.stdout.flush() + +loop = asyncio.get_event_loop() + +print('Pass 1') +# All permutations of discrete task running times must be returned +# by as_completed in the correct order. +task_times = [0, 0.1, 0.2, 0.3, 0.4 ] # 120 permutations +for times in itertools.permutations(task_times): + tasks = [ asyncio.Task(sleeper(t)) for t in times ] + loop.run_until_complete(asyncio.Task(watcher(tasks))) + +print() +print('Pass 2') +# Longer task times, with randomized duplicates. 100 tasks each time. +longer_task_times = [x/10 for x in range(30)] +for i in range(20): + task_times = longer_task_times * 10 + random.shuffle(task_times) + #print('Times', task_times[:500]) + tasks = [ asyncio.Task(sleeper(t)) for t in task_times[:100] ] + loop.run_until_complete(asyncio.Task(watcher(tasks))) + +print() +print('Pass 3') +# Same as pass 2, but with a random processing delay (0 - 0.1s) after +# retrieving each future from as_completed and 200 tasks. This tests whether +# the order that callbacks are triggered is preserved through to the +# as_completed caller. +for i in range(20): + task_times = longer_task_times * 10 + random.shuffle(task_times) + #print('Times', task_times[:200]) + tasks = [ asyncio.Task(sleeper(t)) for t in task_times[:200] ] + loop.run_until_complete(asyncio.Task(watcher(tasks, delay=True))) + +print() +loop.close() diff --git a/thirdparty/asyncio/examples/hello_callback.py b/thirdparty/asyncio/examples/hello_callback.py new file mode 100644 index 0000000..7ccbea1 --- /dev/null +++ b/thirdparty/asyncio/examples/hello_callback.py @@ -0,0 +1,17 @@ +"""Print 'Hello World' every two seconds, using a callback.""" + +import asyncio + + +def print_and_repeat(loop): + print('Hello World') + loop.call_later(2, print_and_repeat, loop) + + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + print_and_repeat(loop) + try: + loop.run_forever() + finally: + loop.close() diff --git a/thirdparty/asyncio/examples/hello_coroutine.py b/thirdparty/asyncio/examples/hello_coroutine.py new file mode 100644 index 0000000..b9347aa --- /dev/null +++ b/thirdparty/asyncio/examples/hello_coroutine.py @@ -0,0 +1,18 @@ +"""Print 'Hello World' every two seconds, using a coroutine.""" + +import asyncio + + +@asyncio.coroutine +def greet_every_two_seconds(): + while True: + print('Hello World') + yield from asyncio.sleep(2) + + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(greet_every_two_seconds()) + finally: + loop.close() diff --git a/thirdparty/asyncio/examples/qspeed.py b/thirdparty/asyncio/examples/qspeed.py new file mode 100644 index 0000000..fcd7116 --- /dev/null +++ b/thirdparty/asyncio/examples/qspeed.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""How fast is the queue implementation?""" + +import time +import asyncio +print(asyncio) + +N_CONSUMERS = 10 +N_PRODUCERS = 1 +N_ITEMS = 100000 # Per producer +Q_SIZE = 1 + +@asyncio.coroutine +def producer(q): + for i in range(N_ITEMS): + yield from q.put(i) + for i in range(N_CONSUMERS): + yield from q.put(None) + +@asyncio.coroutine +def consumer(q): + while True: + i = yield from q.get() + if i is None: + break + +def main(): + q = asyncio.Queue(Q_SIZE) + loop = asyncio.get_event_loop() + consumers = [consumer(q) for _ in range(N_CONSUMERS)] + producers = [producer(q) for _ in range(N_PRODUCERS)] + t0 = time.time() + loop.run_until_complete(asyncio.gather(*consumers, *producers)) + t1 = time.time() + dt = t1 - t0 + print(N_CONSUMERS, 'consumers;', + N_PRODUCERS, 'producers;', + N_ITEMS, 'items/producer;', + Q_SIZE, 'maxsize;', + '%.3f total seconds;' % dt, + '%.3f usec per item.' % (1e6*dt/N_ITEMS/N_PRODUCERS)) + +main() diff --git a/thirdparty/asyncio/examples/shell.py b/thirdparty/asyncio/examples/shell.py new file mode 100644 index 0000000..f934325 --- /dev/null +++ b/thirdparty/asyncio/examples/shell.py @@ -0,0 +1,52 @@ +"""Examples using create_subprocess_exec() and create_subprocess_shell().""" + +import asyncio +import signal +from asyncio.subprocess import PIPE + +@asyncio.coroutine +def cat(loop): + proc = yield from asyncio.create_subprocess_shell("cat", + stdin=PIPE, + stdout=PIPE) + print("pid: %s" % proc.pid) + + message = "Hello World!" + print("cat write: %r" % message) + + stdout, stderr = yield from proc.communicate(message.encode('ascii')) + print("cat read: %r" % stdout.decode('ascii')) + + exitcode = yield from proc.wait() + print("(exit code %s)" % exitcode) + +@asyncio.coroutine +def ls(loop): + proc = yield from asyncio.create_subprocess_exec("ls", + stdout=PIPE) + while True: + line = yield from proc.stdout.readline() + if not line: + break + print("ls>>", line.decode('ascii').rstrip()) + try: + proc.send_signal(signal.SIGINT) + except ProcessLookupError: + pass + +@asyncio.coroutine +def test_call(*args, timeout=None): + proc = yield from asyncio.create_subprocess_exec(*args) + try: + exitcode = yield from asyncio.wait_for(proc.wait(), timeout) + print("%s: exit code %s" % (' '.join(args), exitcode)) + except asyncio.TimeoutError: + print("timeout! (%.1f sec)" % timeout) + proc.kill() + yield from proc.wait() + +loop = asyncio.get_event_loop() +loop.run_until_complete(cat(loop)) +loop.run_until_complete(ls(loop)) +loop.run_until_complete(test_call("bash", "-c", "sleep 3", timeout=1.0)) +loop.close() diff --git a/thirdparty/asyncio/examples/simple_tcp_server.py b/thirdparty/asyncio/examples/simple_tcp_server.py new file mode 100644 index 0000000..5f874ff --- /dev/null +++ b/thirdparty/asyncio/examples/simple_tcp_server.py @@ -0,0 +1,154 @@ +""" +Example of a simple TCP server that is written in (mostly) coroutine +style and uses asyncio.streams.start_server() and +asyncio.streams.open_connection(). + +Note that running this example starts both the TCP server and client +in the same process. It listens on port 12345 on 127.0.0.1, so it will +fail if this port is currently in use. +""" + +import sys +import asyncio +import asyncio.streams + + +class MyServer: + """ + This is just an example of how a TCP server might be potentially + structured. This class has basically 3 methods: start the server, + handle a client, and stop the server. + + Note that you don't have to follow this structure, it is really + just an example or possible starting point. + """ + + def __init__(self): + self.server = None # encapsulates the server sockets + + # this keeps track of all the clients that connected to our + # server. It can be useful in some cases, for instance to + # kill client connections or to broadcast some data to all + # clients... + self.clients = {} # task -> (reader, writer) + + def _accept_client(self, client_reader, client_writer): + """ + This method accepts a new client connection and creates a Task + to handle this client. self.clients is updated to keep track + of the new client. + """ + + # start a new Task to handle this specific client connection + task = asyncio.Task(self._handle_client(client_reader, client_writer)) + self.clients[task] = (client_reader, client_writer) + + def client_done(task): + print("client task done:", task, file=sys.stderr) + del self.clients[task] + + task.add_done_callback(client_done) + + @asyncio.coroutine + def _handle_client(self, client_reader, client_writer): + """ + This method actually does the work to handle the requests for + a specific client. The protocol is line oriented, so there is + a main loop that reads a line with a request and then sends + out one or more lines back to the client with the result. + """ + while True: + data = (yield from client_reader.readline()).decode("utf-8") + if not data: # an empty string means the client disconnected + break + cmd, *args = data.rstrip().split(' ') + if cmd == 'add': + arg1 = float(args[0]) + arg2 = float(args[1]) + retval = arg1 + arg2 + client_writer.write("{!r}\n".format(retval).encode("utf-8")) + elif cmd == 'repeat': + times = int(args[0]) + msg = args[1] + client_writer.write("begin\n".encode("utf-8")) + for idx in range(times): + client_writer.write("{}. {}\n".format(idx+1, msg) + .encode("utf-8")) + client_writer.write("end\n".encode("utf-8")) + else: + print("Bad command {!r}".format(data), file=sys.stderr) + + # This enables us to have flow control in our connection. + yield from client_writer.drain() + + def start(self, loop): + """ + Starts the TCP server, so that it listens on port 12345. + + For each client that connects, the accept_client method gets + called. This method runs the loop until the server sockets + are ready to accept connections. + """ + self.server = loop.run_until_complete( + asyncio.streams.start_server(self._accept_client, + '127.0.0.1', 12345, + loop=loop)) + + def stop(self, loop): + """ + Stops the TCP server, i.e. closes the listening socket(s). + + This method runs the loop until the server sockets are closed. + """ + if self.server is not None: + self.server.close() + loop.run_until_complete(self.server.wait_closed()) + self.server = None + + +def main(): + loop = asyncio.get_event_loop() + + # creates a server and starts listening to TCP connections + server = MyServer() + server.start(loop) + + @asyncio.coroutine + def client(): + reader, writer = yield from asyncio.streams.open_connection( + '127.0.0.1', 12345, loop=loop) + + def send(msg): + print("> " + msg) + writer.write((msg + '\n').encode("utf-8")) + + def recv(): + msgback = (yield from reader.readline()).decode("utf-8").rstrip() + print("< " + msgback) + return msgback + + # send a line + send("add 1 2") + msg = yield from recv() + + send("repeat 5 hello") + msg = yield from recv() + assert msg == 'begin' + while True: + msg = yield from recv() + if msg == 'end': + break + + writer.close() + yield from asyncio.sleep(0.5) + + # creates a client and connects to our server + try: + loop.run_until_complete(client()) + server.stop(loop) + finally: + loop.close() + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/sink.py b/thirdparty/asyncio/examples/sink.py new file mode 100644 index 0000000..d362cbb --- /dev/null +++ b/thirdparty/asyncio/examples/sink.py @@ -0,0 +1,94 @@ +"""Test service that accepts connections and reads all data off them.""" + +import argparse +import os +import sys + +from asyncio import * + +ARGS = argparse.ArgumentParser(description="TCP data sink example.") +ARGS.add_argument( + '--tls', action='store_true', dest='tls', + default=False, help='Use TLS with a self-signed cert') +ARGS.add_argument( + '--iocp', action='store_true', dest='iocp', + default=False, help='Use IOCP event loop (Windows only)') +ARGS.add_argument( + '--host', action='store', dest='host', + default='127.0.0.1', help='Host name') +ARGS.add_argument( + '--port', action='store', dest='port', + default=1111, type=int, help='Port number') +ARGS.add_argument( + '--maxsize', action='store', dest='maxsize', + default=16*1024*1024, type=int, help='Max total data size') + +server = None +args = None + + +def dprint(*args): + print('sink:', *args, file=sys.stderr) + + +class Service(Protocol): + + def connection_made(self, tr): + dprint('connection from', tr.get_extra_info('peername')) + dprint('my socket is', tr.get_extra_info('sockname')) + self.tr = tr + self.total = 0 + + def data_received(self, data): + if data == b'stop': + dprint('stopping server') + server.close() + self.tr.close() + return + self.total += len(data) + dprint('received', len(data), 'bytes; total', self.total) + if self.total > args.maxsize: + dprint('closing due to too much data') + self.tr.close() + + def connection_lost(self, how): + dprint('closed', repr(how)) + + +@coroutine +def start(loop, host, port): + global server + sslctx = None + if args.tls: + import ssl + # TODO: take cert/key from args as well. + here = os.path.join(os.path.dirname(__file__), '..', 'tests') + sslctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslctx.options |= ssl.OP_NO_SSLv2 + sslctx.load_cert_chain( + certfile=os.path.join(here, 'ssl_cert.pem'), + keyfile=os.path.join(here, 'ssl_key.pem')) + + server = yield from loop.create_server(Service, host, port, ssl=sslctx) + dprint('serving TLS' if sslctx else 'serving', + [s.getsockname() for s in server.sockets]) + yield from server.wait_closed() + + +def main(): + global args + args = ARGS.parse_args() + if args.iocp: + from asyncio.windows_events import ProactorEventLoop + loop = ProactorEventLoop() + set_event_loop(loop) + else: + loop = get_event_loop() + try: + loop.run_until_complete(start(loop, args.host, args.port)) + finally: + loop.close() + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/source.py b/thirdparty/asyncio/examples/source.py new file mode 100644 index 0000000..7fd11fb --- /dev/null +++ b/thirdparty/asyncio/examples/source.py @@ -0,0 +1,100 @@ +"""Test client that connects and sends infinite data.""" + +import argparse +import sys + +from asyncio import * +from asyncio import test_utils + + +ARGS = argparse.ArgumentParser(description="TCP data sink example.") +ARGS.add_argument( + '--tls', action='store_true', dest='tls', + default=False, help='Use TLS') +ARGS.add_argument( + '--iocp', action='store_true', dest='iocp', + default=False, help='Use IOCP event loop (Windows only)') +ARGS.add_argument( + '--stop', action='store_true', dest='stop', + default=False, help='Stop the server by sending it b"stop" as data') +ARGS.add_argument( + '--host', action='store', dest='host', + default='127.0.0.1', help='Host name') +ARGS.add_argument( + '--port', action='store', dest='port', + default=1111, type=int, help='Port number') +ARGS.add_argument( + '--size', action='store', dest='size', + default=16*1024, type=int, help='Data size') + +args = None + + +def dprint(*args): + print('source:', *args, file=sys.stderr) + + +class Client(Protocol): + + total = 0 + + def connection_made(self, tr): + dprint('connecting to', tr.get_extra_info('peername')) + dprint('my socket is', tr.get_extra_info('sockname')) + self.tr = tr + self.lost = False + self.loop = get_event_loop() + self.waiter = Future() + if args.stop: + self.tr.write(b'stop') + self.tr.close() + else: + self.data = b'x'*args.size + self.write_some_data() + + def write_some_data(self): + if self.lost: + dprint('lost already') + return + data = self.data + size = len(data) + self.total += size + dprint('writing', size, 'bytes; total', self.total) + self.tr.write(data) + self.loop.call_soon(self.write_some_data) + + def connection_lost(self, exc): + dprint('lost connection', repr(exc)) + self.lost = True + self.waiter.set_result(None) + + +@coroutine +def start(loop, host, port): + sslctx = None + if args.tls: + sslctx = test_utils.dummy_ssl_context() + tr, pr = yield from loop.create_connection(Client, host, port, + ssl=sslctx) + dprint('tr =', tr) + dprint('pr =', pr) + yield from pr.waiter + + +def main(): + global args + args = ARGS.parse_args() + if args.iocp: + from asyncio.windows_events import ProactorEventLoop + loop = ProactorEventLoop() + set_event_loop(loop) + else: + loop = get_event_loop() + try: + loop.run_until_complete(start(loop, args.host, args.port)) + finally: + loop.close() + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/source1.py b/thirdparty/asyncio/examples/source1.py new file mode 100644 index 0000000..6802e96 --- /dev/null +++ b/thirdparty/asyncio/examples/source1.py @@ -0,0 +1,98 @@ +"""Like source.py, but uses streams.""" + +import argparse +import sys + +from asyncio import * +from asyncio import test_utils + +ARGS = argparse.ArgumentParser(description="TCP data sink example.") +ARGS.add_argument( + '--tls', action='store_true', dest='tls', + default=False, help='Use TLS') +ARGS.add_argument( + '--iocp', action='store_true', dest='iocp', + default=False, help='Use IOCP event loop (Windows only)') +ARGS.add_argument( + '--stop', action='store_true', dest='stop', + default=False, help='Stop the server by sending it b"stop" as data') +ARGS.add_argument( + '--host', action='store', dest='host', + default='127.0.0.1', help='Host name') +ARGS.add_argument( + '--port', action='store', dest='port', + default=1111, type=int, help='Port number') +ARGS.add_argument( + '--size', action='store', dest='size', + default=16*1024, type=int, help='Data size') + + +class Debug: + """A clever little class that suppresses repetitive messages.""" + + overwriting = False + label = 'stream1:' + + def print(self, *args): + if self.overwriting: + print(file=sys.stderr) + self.overwriting = 0 + print(self.label, *args, file=sys.stderr) + + def oprint(self, *args): + self.overwriting += 1 + end = '\n' + if self.overwriting >= 3: + if self.overwriting == 3: + print(self.label, '[...]', file=sys.stderr) + end = '\r' + print(self.label, *args, file=sys.stderr, end=end, flush=True) + + +@coroutine +def start(loop, args): + d = Debug() + total = 0 + sslctx = None + if args.tls: + d.print('using dummy SSLContext') + sslctx = test_utils.dummy_ssl_context() + r, w = yield from open_connection(args.host, args.port, ssl=sslctx) + d.print('r =', r) + d.print('w =', w) + if args.stop: + w.write(b'stop') + w.close() + else: + size = args.size + data = b'x'*size + try: + while True: + total += size + d.oprint('writing', size, 'bytes; total', total) + w.write(data) + f = w.drain() + if f: + d.print('pausing') + yield from f + except (ConnectionResetError, BrokenPipeError) as exc: + d.print('caught', repr(exc)) + + +def main(): + global args + args = ARGS.parse_args() + if args.iocp: + from asyncio.windows_events import ProactorEventLoop + loop = ProactorEventLoop() + set_event_loop(loop) + else: + loop = get_event_loop() + try: + loop.run_until_complete(start(loop, args)) + finally: + loop.close() + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/stacks.py b/thirdparty/asyncio/examples/stacks.py new file mode 100644 index 0000000..0b7e0b2 --- /dev/null +++ b/thirdparty/asyncio/examples/stacks.py @@ -0,0 +1,44 @@ +"""Crude demo for print_stack().""" + + +from asyncio import * + + +@coroutine +def helper(r): + print('--- helper ---') + for t in Task.all_tasks(): + t.print_stack() + print('--- end helper ---') + line = yield from r.readline() + 1/0 + return line + +def doit(): + l = get_event_loop() + lr = l.run_until_complete + r, w = lr(open_connection('python.org', 80)) + t1 = async(helper(r)) + for t in Task.all_tasks(): t.print_stack() + print('---') + l._run_once() + for t in Task.all_tasks(): t.print_stack() + print('---') + w.write(b'GET /\r\n') + w.write_eof() + try: + lr(t1) + except Exception as e: + print('catching', e) + finally: + for t in Task.all_tasks(): + t.print_stack() + l.close() + + +def main(): + doit() + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/subprocess_attach_read_pipe.py b/thirdparty/asyncio/examples/subprocess_attach_read_pipe.py new file mode 100644 index 0000000..d8a6242 --- /dev/null +++ b/thirdparty/asyncio/examples/subprocess_attach_read_pipe.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Example showing how to attach a read pipe to a subprocess.""" +import asyncio +import os, sys + +code = """ +import os, sys +fd = int(sys.argv[1]) +os.write(fd, b'data') +os.close(fd) +""" + +loop = asyncio.get_event_loop() + +@asyncio.coroutine +def task(): + rfd, wfd = os.pipe() + args = [sys.executable, '-c', code, str(wfd)] + + pipe = open(rfd, 'rb', 0) + reader = asyncio.StreamReader(loop=loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=loop) + transport, _ = yield from loop.connect_read_pipe(lambda: protocol, pipe) + + proc = yield from asyncio.create_subprocess_exec(*args, pass_fds={wfd}) + yield from proc.wait() + + os.close(wfd) + data = yield from reader.read() + print("read = %r" % data.decode()) + +loop.run_until_complete(task()) +loop.close() diff --git a/thirdparty/asyncio/examples/subprocess_attach_write_pipe.py b/thirdparty/asyncio/examples/subprocess_attach_write_pipe.py new file mode 100644 index 0000000..c4e099f --- /dev/null +++ b/thirdparty/asyncio/examples/subprocess_attach_write_pipe.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Example showing how to attach a write pipe to a subprocess.""" +import asyncio +import os, sys +from asyncio import subprocess + +code = """ +import os, sys +fd = int(sys.argv[1]) +data = os.read(fd, 1024) +sys.stdout.buffer.write(data) +""" + +loop = asyncio.get_event_loop() + +@asyncio.coroutine +def task(): + rfd, wfd = os.pipe() + args = [sys.executable, '-c', code, str(rfd)] + proc = yield from asyncio.create_subprocess_exec( + *args, + pass_fds={rfd}, + stdout=subprocess.PIPE) + + pipe = open(wfd, 'wb', 0) + transport, _ = yield from loop.connect_write_pipe(asyncio.Protocol, + pipe) + transport.write(b'data') + + stdout, stderr = yield from proc.communicate() + print("stdout = %r" % stdout.decode()) + transport.close() + +loop.run_until_complete(task()) +loop.close() diff --git a/thirdparty/asyncio/examples/subprocess_shell.py b/thirdparty/asyncio/examples/subprocess_shell.py new file mode 100644 index 0000000..745cb64 --- /dev/null +++ b/thirdparty/asyncio/examples/subprocess_shell.py @@ -0,0 +1,87 @@ +"""Example writing to and reading from a subprocess at the same time using +tasks.""" + +import asyncio +import os +from asyncio.subprocess import PIPE + + +@asyncio.coroutine +def send_input(writer, input): + try: + for line in input: + print('sending', len(line), 'bytes') + writer.write(line) + d = writer.drain() + if d: + print('pause writing') + yield from d + print('resume writing') + writer.close() + except BrokenPipeError: + print('stdin: broken pipe error') + except ConnectionResetError: + print('stdin: connection reset error') + +@asyncio.coroutine +def log_errors(reader): + while True: + line = yield from reader.readline() + if not line: + break + print('ERROR', repr(line)) + +@asyncio.coroutine +def read_stdout(stdout): + while True: + line = yield from stdout.readline() + print('received', repr(line)) + if not line: + break + +@asyncio.coroutine +def start(cmd, input=None, **kwds): + kwds['stdout'] = PIPE + kwds['stderr'] = PIPE + if input is None and 'stdin' not in kwds: + kwds['stdin'] = None + else: + kwds['stdin'] = PIPE + proc = yield from asyncio.create_subprocess_shell(cmd, **kwds) + + tasks = [] + if input is not None: + tasks.append(send_input(proc.stdin, input)) + else: + print('No stdin') + if proc.stderr is not None: + tasks.append(log_errors(proc.stderr)) + else: + print('No stderr') + if proc.stdout is not None: + tasks.append(read_stdout(proc.stdout)) + else: + print('No stdout') + + if tasks: + # feed stdin while consuming stdout to avoid hang + # when stdin pipe is full + yield from asyncio.wait(tasks) + + exitcode = yield from proc.wait() + print("exit code: %s" % exitcode) + + +def main(): + if os.name == 'nt': + loop = asyncio.ProactorEventLoop() + asyncio.set_event_loop(loop) + else: + loop = asyncio.get_event_loop() + loop.run_until_complete(start( + 'sleep 2; wc', input=[b'foo bar baz\n'*300 for i in range(100)])) + loop.close() + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/tcp_echo.py b/thirdparty/asyncio/examples/tcp_echo.py new file mode 100644 index 0000000..d743242 --- /dev/null +++ b/thirdparty/asyncio/examples/tcp_echo.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""TCP echo server example.""" +import argparse +import asyncio +import sys +try: + import signal +except ImportError: + signal = None + + +class EchoServer(asyncio.Protocol): + + TIMEOUT = 5.0 + + def timeout(self): + print('connection timeout, closing.') + self.transport.close() + + def connection_made(self, transport): + print('connection made') + self.transport = transport + + # start 5 seconds timeout timer + self.h_timeout = asyncio.get_event_loop().call_later( + self.TIMEOUT, self.timeout) + + def data_received(self, data): + print('data received: ', data.decode()) + self.transport.write(b'Re: ' + data) + + # restart timeout timer + self.h_timeout.cancel() + self.h_timeout = asyncio.get_event_loop().call_later( + self.TIMEOUT, self.timeout) + + def eof_received(self): + pass + + def connection_lost(self, exc): + print('connection lost:', exc) + self.h_timeout.cancel() + + +class EchoClient(asyncio.Protocol): + + message = 'This is the message. It will be echoed.' + + def connection_made(self, transport): + self.transport = transport + self.transport.write(self.message.encode()) + print('data sent:', self.message) + + def data_received(self, data): + print('data received:', data) + + # disconnect after 10 seconds + asyncio.get_event_loop().call_later(10.0, self.transport.close) + + def eof_received(self): + pass + + def connection_lost(self, exc): + print('connection lost:', exc) + asyncio.get_event_loop().stop() + + +def start_client(loop, host, port): + t = asyncio.Task(loop.create_connection(EchoClient, host, port)) + loop.run_until_complete(t) + + +def start_server(loop, host, port): + f = loop.create_server(EchoServer, host, port) + return loop.run_until_complete(f) + + +ARGS = argparse.ArgumentParser(description="TCP Echo example.") +ARGS.add_argument( + '--server', action="store_true", dest='server', + default=False, help='Run tcp server') +ARGS.add_argument( + '--client', action="store_true", dest='client', + default=False, help='Run tcp client') +ARGS.add_argument( + '--host', action="store", dest='host', + default='127.0.0.1', help='Host name') +ARGS.add_argument( + '--port', action="store", dest='port', + default=9999, type=int, help='Port number') +ARGS.add_argument( + '--iocp', action="store_true", dest='iocp', + default=False, help='Use IOCP event loop') + + +if __name__ == '__main__': + args = ARGS.parse_args() + + if ':' in args.host: + args.host, port = args.host.split(':', 1) + args.port = int(port) + + if (not (args.server or args.client)) or (args.server and args.client): + print('Please specify --server or --client\n') + ARGS.print_help() + else: + if args.iocp: + from asyncio import windows_events + loop = windows_events.ProactorEventLoop() + asyncio.set_event_loop(loop) + else: + loop = asyncio.get_event_loop() + print ('Using backend: {0}'.format(loop.__class__.__name__)) + + if signal is not None and sys.platform != 'win32': + loop.add_signal_handler(signal.SIGINT, loop.stop) + + if args.server: + server = start_server(loop, args.host, args.port) + else: + start_client(loop, args.host, args.port) + + try: + loop.run_forever() + finally: + if args.server: + server.close() + loop.close() diff --git a/thirdparty/asyncio/examples/timing_tcp_server.py b/thirdparty/asyncio/examples/timing_tcp_server.py new file mode 100644 index 0000000..3fcdc97 --- /dev/null +++ b/thirdparty/asyncio/examples/timing_tcp_server.py @@ -0,0 +1,168 @@ +""" +A variant of simple_tcp_server.py that measures the time it takes to +send N messages for a range of N. (This was O(N**2) in a previous +version of asyncio.) + +Note that running this example starts both the TCP server and client +in the same process. It listens on port 1234 on 127.0.0.1, so it will +fail if this port is currently in use. +""" + +import sys +import time +import random + +import asyncio +import asyncio.streams + + +class MyServer: + """ + This is just an example of how a TCP server might be potentially + structured. This class has basically 3 methods: start the server, + handle a client, and stop the server. + + Note that you don't have to follow this structure, it is really + just an example or possible starting point. + """ + + def __init__(self): + self.server = None # encapsulates the server sockets + + # this keeps track of all the clients that connected to our + # server. It can be useful in some cases, for instance to + # kill client connections or to broadcast some data to all + # clients... + self.clients = {} # task -> (reader, writer) + + def _accept_client(self, client_reader, client_writer): + """ + This method accepts a new client connection and creates a Task + to handle this client. self.clients is updated to keep track + of the new client. + """ + + # start a new Task to handle this specific client connection + task = asyncio.Task(self._handle_client(client_reader, client_writer)) + self.clients[task] = (client_reader, client_writer) + + def client_done(task): + print("client task done:", task, file=sys.stderr) + del self.clients[task] + + task.add_done_callback(client_done) + + @asyncio.coroutine + def _handle_client(self, client_reader, client_writer): + """ + This method actually does the work to handle the requests for + a specific client. The protocol is line oriented, so there is + a main loop that reads a line with a request and then sends + out one or more lines back to the client with the result. + """ + while True: + data = (yield from client_reader.readline()).decode("utf-8") + if not data: # an empty string means the client disconnected + break + cmd, *args = data.rstrip().split(' ') + if cmd == 'add': + arg1 = float(args[0]) + arg2 = float(args[1]) + retval = arg1 + arg2 + client_writer.write("{!r}\n".format(retval).encode("utf-8")) + elif cmd == 'repeat': + times = int(args[0]) + msg = args[1] + client_writer.write("begin\n".encode("utf-8")) + for idx in range(times): + client_writer.write("{}. {}\n".format( + idx+1, msg + 'x'*random.randint(10, 50)) + .encode("utf-8")) + client_writer.write("end\n".encode("utf-8")) + else: + print("Bad command {!r}".format(data), file=sys.stderr) + + # This enables us to have flow control in our connection. + yield from client_writer.drain() + + def start(self, loop): + """ + Starts the TCP server, so that it listens on port 1234. + + For each client that connects, the accept_client method gets + called. This method runs the loop until the server sockets + are ready to accept connections. + """ + self.server = loop.run_until_complete( + asyncio.streams.start_server(self._accept_client, + '127.0.0.1', 12345, + loop=loop)) + + def stop(self, loop): + """ + Stops the TCP server, i.e. closes the listening socket(s). + + This method runs the loop until the server sockets are closed. + """ + if self.server is not None: + self.server.close() + loop.run_until_complete(self.server.wait_closed()) + self.server = None + + +def main(): + loop = asyncio.get_event_loop() + + # creates a server and starts listening to TCP connections + server = MyServer() + server.start(loop) + + @asyncio.coroutine + def client(): + reader, writer = yield from asyncio.streams.open_connection( + '127.0.0.1', 12345, loop=loop) + + def send(msg): + print("> " + msg) + writer.write((msg + '\n').encode("utf-8")) + + def recv(): + msgback = (yield from reader.readline()).decode("utf-8").rstrip() + print("< " + msgback) + return msgback + + # send a line + send("add 1 2") + msg = yield from recv() + + Ns = list(range(100, 100000, 10000)) + times = [] + + for N in Ns: + t0 = time.time() + send("repeat {} hello world ".format(N)) + msg = yield from recv() + assert msg == 'begin' + while True: + msg = (yield from reader.readline()).decode("utf-8").rstrip() + if msg == 'end': + break + t1 = time.time() + dt = t1 - t0 + print("Time taken: {:.3f} seconds ({:.6f} per repetition)" + .format(dt, dt/N)) + times.append(dt) + + writer.close() + yield from asyncio.sleep(0.5) + + # creates a client and connects to our server + try: + loop.run_until_complete(client()) + server.stop(loop) + finally: + loop.close() + + +if __name__ == '__main__': + main() diff --git a/thirdparty/asyncio/examples/udp_echo.py b/thirdparty/asyncio/examples/udp_echo.py new file mode 100644 index 0000000..93ac7e6 --- /dev/null +++ b/thirdparty/asyncio/examples/udp_echo.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""UDP echo example.""" +import argparse +import sys +import asyncio +try: + import signal +except ImportError: + signal = None + + +class MyServerUdpEchoProtocol: + + def connection_made(self, transport): + print('start', transport) + self.transport = transport + + def datagram_received(self, data, addr): + print('Data received:', data, addr) + self.transport.sendto(data, addr) + + def error_received(self, exc): + print('Error received:', exc) + + def connection_lost(self, exc): + print('stop', exc) + + +class MyClientUdpEchoProtocol: + + message = 'This is the message. It will be echoed.' + + def connection_made(self, transport): + self.transport = transport + print('sending "{}"'.format(self.message)) + self.transport.sendto(self.message.encode()) + print('waiting to receive') + + def datagram_received(self, data, addr): + print('received "{}"'.format(data.decode())) + self.transport.close() + + def error_received(self, exc): + print('Error received:', exc) + + def connection_lost(self, exc): + print('closing transport', exc) + loop = asyncio.get_event_loop() + loop.stop() + + +def start_server(loop, addr): + t = asyncio.Task(loop.create_datagram_endpoint( + MyServerUdpEchoProtocol, local_addr=addr)) + transport, server = loop.run_until_complete(t) + return transport + + +def start_client(loop, addr): + t = asyncio.Task(loop.create_datagram_endpoint( + MyClientUdpEchoProtocol, remote_addr=addr)) + loop.run_until_complete(t) + + +ARGS = argparse.ArgumentParser(description="UDP Echo example.") +ARGS.add_argument( + '--server', action="store_true", dest='server', + default=False, help='Run udp server') +ARGS.add_argument( + '--client', action="store_true", dest='client', + default=False, help='Run udp client') +ARGS.add_argument( + '--host', action="store", dest='host', + default='127.0.0.1', help='Host name') +ARGS.add_argument( + '--port', action="store", dest='port', + default=9999, type=int, help='Port number') + + +if __name__ == '__main__': + args = ARGS.parse_args() + if ':' in args.host: + args.host, port = args.host.split(':', 1) + args.port = int(port) + + if (not (args.server or args.client)) or (args.server and args.client): + print('Please specify --server or --client\n') + ARGS.print_help() + else: + loop = asyncio.get_event_loop() + if signal is not None: + loop.add_signal_handler(signal.SIGINT, loop.stop) + + if '--server' in sys.argv: + server = start_server(loop, (args.host, args.port)) + else: + start_client(loop, (args.host, args.port)) + + try: + loop.run_forever() + finally: + if '--server' in sys.argv: + server.close() + loop.close() diff --git a/thirdparty/asyncio/overlapped.c b/thirdparty/asyncio/overlapped.c new file mode 100644 index 0000000..f85e5bc --- /dev/null +++ b/thirdparty/asyncio/overlapped.c @@ -0,0 +1,1346 @@ +/* + * Support for overlapped IO + * + * Some code borrowed from Modules/_winapi.c of CPython + */ + +/* XXX check overflow and DWORD <-> Py_ssize_t conversions + Check itemsize */ + +#include "Python.h" +#include "structmember.h" + +#define WINDOWS_LEAN_AND_MEAN +#include +#include +#include + +#if defined(MS_WIN32) && !defined(MS_WIN64) +# define F_POINTER "k" +# define T_POINTER T_ULONG +#else +# define F_POINTER "K" +# define T_POINTER T_ULONGLONG +#endif + +/* Compatibility with Python 3.3 */ +#if PY_VERSION_HEX < 0x03040000 +# define PyMem_RawMalloc PyMem_Malloc +# define PyMem_RawFree PyMem_Free +#endif + +#define F_HANDLE F_POINTER +#define F_ULONG_PTR F_POINTER +#define F_DWORD "k" +#define F_BOOL "i" +#define F_UINT "I" + +#define T_HANDLE T_POINTER + +enum {TYPE_NONE, TYPE_NOT_STARTED, TYPE_READ, TYPE_WRITE, TYPE_ACCEPT, + TYPE_CONNECT, TYPE_DISCONNECT, TYPE_CONNECT_NAMED_PIPE, + TYPE_WAIT_NAMED_PIPE_AND_CONNECT}; + +typedef struct { + PyObject_HEAD + OVERLAPPED overlapped; + /* For convenience, we store the file handle too */ + HANDLE handle; + /* Error returned by last method call */ + DWORD error; + /* Type of operation */ + DWORD type; + union { + /* Buffer used for reading: TYPE_READ and TYPE_ACCEPT */ + PyObject *read_buffer; + /* Buffer used for writing: TYPE_WRITE */ + Py_buffer write_buffer; + }; +} OverlappedObject; + +/* + * Map Windows error codes to subclasses of OSError + */ + +static PyObject * +SetFromWindowsErr(DWORD err) +{ + PyObject *exception_type; + + if (err == 0) + err = GetLastError(); + switch (err) { + case ERROR_CONNECTION_REFUSED: + exception_type = PyExc_ConnectionRefusedError; + break; + case ERROR_CONNECTION_ABORTED: + exception_type = PyExc_ConnectionAbortedError; + break; + default: + exception_type = PyExc_OSError; + } + return PyErr_SetExcFromWindowsErr(exception_type, err); +} + +/* + * Some functions should be loaded at runtime + */ + +static LPFN_ACCEPTEX Py_AcceptEx = NULL; +static LPFN_CONNECTEX Py_ConnectEx = NULL; +static LPFN_DISCONNECTEX Py_DisconnectEx = NULL; +static BOOL (CALLBACK *Py_CancelIoEx)(HANDLE, LPOVERLAPPED) = NULL; + +#define GET_WSA_POINTER(s, x) \ + (SOCKET_ERROR != WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, \ + &Guid##x, sizeof(Guid##x), &Py_##x, \ + sizeof(Py_##x), &dwBytes, NULL, NULL)) + +static int +initialize_function_pointers(void) +{ + GUID GuidAcceptEx = WSAID_ACCEPTEX; + GUID GuidConnectEx = WSAID_CONNECTEX; + GUID GuidDisconnectEx = WSAID_DISCONNECTEX; + HINSTANCE hKernel32; + SOCKET s; + DWORD dwBytes; + + s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (s == INVALID_SOCKET) { + SetFromWindowsErr(WSAGetLastError()); + return -1; + } + + if (!GET_WSA_POINTER(s, AcceptEx) || + !GET_WSA_POINTER(s, ConnectEx) || + !GET_WSA_POINTER(s, DisconnectEx)) + { + closesocket(s); + SetFromWindowsErr(WSAGetLastError()); + return -1; + } + + closesocket(s); + + /* On WinXP we will have Py_CancelIoEx == NULL */ + hKernel32 = GetModuleHandle("KERNEL32"); + *(FARPROC *)&Py_CancelIoEx = GetProcAddress(hKernel32, "CancelIoEx"); + return 0; +} + +/* + * Completion port stuff + */ + +PyDoc_STRVAR( + CreateIoCompletionPort_doc, + "CreateIoCompletionPort(handle, port, key, concurrency) -> port\n\n" + "Create a completion port or register a handle with a port."); + +static PyObject * +overlapped_CreateIoCompletionPort(PyObject *self, PyObject *args) +{ + HANDLE FileHandle; + HANDLE ExistingCompletionPort; + ULONG_PTR CompletionKey; + DWORD NumberOfConcurrentThreads; + HANDLE ret; + + if (!PyArg_ParseTuple(args, F_HANDLE F_HANDLE F_ULONG_PTR F_DWORD, + &FileHandle, &ExistingCompletionPort, &CompletionKey, + &NumberOfConcurrentThreads)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = CreateIoCompletionPort(FileHandle, ExistingCompletionPort, + CompletionKey, NumberOfConcurrentThreads); + Py_END_ALLOW_THREADS + + if (ret == NULL) + return SetFromWindowsErr(0); + return Py_BuildValue(F_HANDLE, ret); +} + +PyDoc_STRVAR( + GetQueuedCompletionStatus_doc, + "GetQueuedCompletionStatus(port, msecs) -> (err, bytes, key, address)\n\n" + "Get a message from completion port. Wait for up to msecs milliseconds."); + +static PyObject * +overlapped_GetQueuedCompletionStatus(PyObject *self, PyObject *args) +{ + HANDLE CompletionPort = NULL; + DWORD NumberOfBytes = 0; + ULONG_PTR CompletionKey = 0; + OVERLAPPED *Overlapped = NULL; + DWORD Milliseconds; + DWORD err; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE F_DWORD, + &CompletionPort, &Milliseconds)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = GetQueuedCompletionStatus(CompletionPort, &NumberOfBytes, + &CompletionKey, &Overlapped, Milliseconds); + Py_END_ALLOW_THREADS + + err = ret ? ERROR_SUCCESS : GetLastError(); + if (Overlapped == NULL) { + if (err == WAIT_TIMEOUT) + Py_RETURN_NONE; + else + return SetFromWindowsErr(err); + } + return Py_BuildValue(F_DWORD F_DWORD F_ULONG_PTR F_POINTER, + err, NumberOfBytes, CompletionKey, Overlapped); +} + +PyDoc_STRVAR( + PostQueuedCompletionStatus_doc, + "PostQueuedCompletionStatus(port, bytes, key, address) -> None\n\n" + "Post a message to completion port."); + +static PyObject * +overlapped_PostQueuedCompletionStatus(PyObject *self, PyObject *args) +{ + HANDLE CompletionPort; + DWORD NumberOfBytes; + ULONG_PTR CompletionKey; + OVERLAPPED *Overlapped; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE F_DWORD F_ULONG_PTR F_POINTER, + &CompletionPort, &NumberOfBytes, &CompletionKey, + &Overlapped)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = PostQueuedCompletionStatus(CompletionPort, NumberOfBytes, + CompletionKey, Overlapped); + Py_END_ALLOW_THREADS + + if (!ret) + return SetFromWindowsErr(0); + Py_RETURN_NONE; +} + +/* + * Wait for a handle + */ + +struct PostCallbackData { + HANDLE CompletionPort; + LPOVERLAPPED Overlapped; +}; + +static VOID CALLBACK +PostToQueueCallback(PVOID lpParameter, BOOL TimerOrWaitFired) +{ + struct PostCallbackData *p = (struct PostCallbackData*) lpParameter; + + PostQueuedCompletionStatus(p->CompletionPort, TimerOrWaitFired, + 0, p->Overlapped); + /* ignore possible error! */ + PyMem_RawFree(p); +} + +PyDoc_STRVAR( + RegisterWaitWithQueue_doc, + "RegisterWaitWithQueue(Object, CompletionPort, Overlapped, Timeout)\n" + " -> WaitHandle\n\n" + "Register wait for Object; when complete CompletionPort is notified.\n"); + +static PyObject * +overlapped_RegisterWaitWithQueue(PyObject *self, PyObject *args) +{ + HANDLE NewWaitObject; + HANDLE Object; + ULONG Milliseconds; + struct PostCallbackData data, *pdata; + + if (!PyArg_ParseTuple(args, F_HANDLE F_HANDLE F_POINTER F_DWORD, + &Object, + &data.CompletionPort, + &data.Overlapped, + &Milliseconds)) + return NULL; + + /* Use PyMem_RawMalloc() rather than PyMem_Malloc(), since + PostToQueueCallback() will call PyMem_Free() from a new C thread + which doesn't hold the GIL. */ + pdata = PyMem_RawMalloc(sizeof(struct PostCallbackData)); + if (pdata == NULL) + return SetFromWindowsErr(0); + + *pdata = data; + + if (!RegisterWaitForSingleObject( + &NewWaitObject, Object, (WAITORTIMERCALLBACK)PostToQueueCallback, + pdata, Milliseconds, + WT_EXECUTEINWAITTHREAD | WT_EXECUTEONLYONCE)) + { + PyMem_RawFree(pdata); + return SetFromWindowsErr(0); + } + + return Py_BuildValue(F_HANDLE, NewWaitObject); +} + +PyDoc_STRVAR( + UnregisterWait_doc, + "UnregisterWait(WaitHandle) -> None\n\n" + "Unregister wait handle.\n"); + +static PyObject * +overlapped_UnregisterWait(PyObject *self, PyObject *args) +{ + HANDLE WaitHandle; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE, &WaitHandle)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = UnregisterWait(WaitHandle); + Py_END_ALLOW_THREADS + + if (!ret) + return SetFromWindowsErr(0); + Py_RETURN_NONE; +} + +PyDoc_STRVAR( + UnregisterWaitEx_doc, + "UnregisterWaitEx(WaitHandle, Event) -> None\n\n" + "Unregister wait handle.\n"); + +static PyObject * +overlapped_UnregisterWaitEx(PyObject *self, PyObject *args) +{ + HANDLE WaitHandle, Event; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE F_HANDLE, &WaitHandle, &Event)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = UnregisterWaitEx(WaitHandle, Event); + Py_END_ALLOW_THREADS + + if (!ret) + return SetFromWindowsErr(0); + Py_RETURN_NONE; +} + +/* + * Event functions -- currently only used by tests + */ + +PyDoc_STRVAR( + CreateEvent_doc, + "CreateEvent(EventAttributes, ManualReset, InitialState, Name)" + " -> Handle\n\n" + "Create an event. EventAttributes must be None.\n"); + +static PyObject * +overlapped_CreateEvent(PyObject *self, PyObject *args) +{ + PyObject *EventAttributes; + BOOL ManualReset; + BOOL InitialState; + Py_UNICODE *Name; + HANDLE Event; + + if (!PyArg_ParseTuple(args, "O" F_BOOL F_BOOL "Z", + &EventAttributes, &ManualReset, + &InitialState, &Name)) + return NULL; + + if (EventAttributes != Py_None) { + PyErr_SetString(PyExc_ValueError, "EventAttributes must be None"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + Event = CreateEventW(NULL, ManualReset, InitialState, Name); + Py_END_ALLOW_THREADS + + if (Event == NULL) + return SetFromWindowsErr(0); + return Py_BuildValue(F_HANDLE, Event); +} + +PyDoc_STRVAR( + SetEvent_doc, + "SetEvent(Handle) -> None\n\n" + "Set event.\n"); + +static PyObject * +overlapped_SetEvent(PyObject *self, PyObject *args) +{ + HANDLE Handle; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE, &Handle)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = SetEvent(Handle); + Py_END_ALLOW_THREADS + + if (!ret) + return SetFromWindowsErr(0); + Py_RETURN_NONE; +} + +PyDoc_STRVAR( + ResetEvent_doc, + "ResetEvent(Handle) -> None\n\n" + "Reset event.\n"); + +static PyObject * +overlapped_ResetEvent(PyObject *self, PyObject *args) +{ + HANDLE Handle; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE, &Handle)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = ResetEvent(Handle); + Py_END_ALLOW_THREADS + + if (!ret) + return SetFromWindowsErr(0); + Py_RETURN_NONE; +} + +/* + * Bind socket handle to local port without doing slow getaddrinfo() + */ + +PyDoc_STRVAR( + BindLocal_doc, + "BindLocal(handle, family) -> None\n\n" + "Bind a socket handle to an arbitrary local port.\n" + "family should AF_INET or AF_INET6.\n"); + +static PyObject * +overlapped_BindLocal(PyObject *self, PyObject *args) +{ + SOCKET Socket; + int Family; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "i", &Socket, &Family)) + return NULL; + + if (Family == AF_INET) { + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = 0; + addr.sin_addr.S_un.S_addr = INADDR_ANY; + ret = bind(Socket, (SOCKADDR*)&addr, sizeof(addr)) != SOCKET_ERROR; + } else if (Family == AF_INET6) { + struct sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_port = 0; + addr.sin6_addr = in6addr_any; + ret = bind(Socket, (SOCKADDR*)&addr, sizeof(addr)) != SOCKET_ERROR; + } else { + PyErr_SetString(PyExc_ValueError, "expected tuple of length 2 or 4"); + return NULL; + } + + if (!ret) + return SetFromWindowsErr(WSAGetLastError()); + Py_RETURN_NONE; +} + +/* + * Windows equivalent of os.strerror() -- compare _ctypes/callproc.c + */ + +PyDoc_STRVAR( + FormatMessage_doc, + "FormatMessage(error_code) -> error_message\n\n" + "Return error message for an error code."); + +static PyObject * +overlapped_FormatMessage(PyObject *ignore, PyObject *args) +{ + DWORD code, n; + WCHAR *lpMsgBuf; + PyObject *res; + + if (!PyArg_ParseTuple(args, F_DWORD, &code)) + return NULL; + + n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + code, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR) &lpMsgBuf, + 0, + NULL); + if (n) { + while (iswspace(lpMsgBuf[n-1])) + --n; + lpMsgBuf[n] = L'\0'; + res = Py_BuildValue("u", lpMsgBuf); + } else { + res = PyUnicode_FromFormat("unknown error code %u", code); + } + LocalFree(lpMsgBuf); + return res; +} + + +/* + * Mark operation as completed - used when reading produces ERROR_BROKEN_PIPE + */ + +static void +mark_as_completed(OVERLAPPED *ov) +{ + ov->Internal = 0; + if (ov->hEvent != NULL) + SetEvent(ov->hEvent); +} + +/* + * A Python object wrapping an OVERLAPPED structure and other useful data + * for overlapped I/O + */ + +PyDoc_STRVAR( + Overlapped_doc, + "Overlapped object"); + +static PyObject * +Overlapped_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + OverlappedObject *self; + HANDLE event = INVALID_HANDLE_VALUE; + static char *kwlist[] = {"event", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|" F_HANDLE, kwlist, &event)) + return NULL; + + if (event == INVALID_HANDLE_VALUE) { + event = CreateEvent(NULL, TRUE, FALSE, NULL); + if (event == NULL) + return SetFromWindowsErr(0); + } + + self = PyObject_New(OverlappedObject, type); + if (self == NULL) { + if (event != NULL) + CloseHandle(event); + return NULL; + } + + self->handle = NULL; + self->error = 0; + self->type = TYPE_NONE; + self->read_buffer = NULL; + memset(&self->overlapped, 0, sizeof(OVERLAPPED)); + memset(&self->write_buffer, 0, sizeof(Py_buffer)); + if (event) + self->overlapped.hEvent = event; + return (PyObject *)self; +} + +static void +Overlapped_dealloc(OverlappedObject *self) +{ + DWORD bytes; + DWORD olderr = GetLastError(); + BOOL wait = FALSE; + BOOL ret; + + if (!HasOverlappedIoCompleted(&self->overlapped) && + self->type != TYPE_NOT_STARTED) + { + if (Py_CancelIoEx && Py_CancelIoEx(self->handle, &self->overlapped)) + wait = TRUE; + + Py_BEGIN_ALLOW_THREADS + ret = GetOverlappedResult(self->handle, &self->overlapped, + &bytes, wait); + Py_END_ALLOW_THREADS + + switch (ret ? ERROR_SUCCESS : GetLastError()) { + case ERROR_SUCCESS: + case ERROR_NOT_FOUND: + case ERROR_OPERATION_ABORTED: + break; + default: + PyErr_Format( + PyExc_RuntimeError, + "%R still has pending operation at " + "deallocation, the process may crash", self); + PyErr_WriteUnraisable(NULL); + } + } + + if (self->overlapped.hEvent != NULL) + CloseHandle(self->overlapped.hEvent); + + switch (self->type) { + case TYPE_READ: + case TYPE_ACCEPT: + Py_CLEAR(self->read_buffer); + break; + case TYPE_WRITE: + if (self->write_buffer.obj) + PyBuffer_Release(&self->write_buffer); + break; + } + PyObject_Del(self); + SetLastError(olderr); +} + +PyDoc_STRVAR( + Overlapped_cancel_doc, + "cancel() -> None\n\n" + "Cancel overlapped operation"); + +static PyObject * +Overlapped_cancel(OverlappedObject *self) +{ + BOOL ret = TRUE; + + if (self->type == TYPE_NOT_STARTED + || self->type == TYPE_WAIT_NAMED_PIPE_AND_CONNECT) + Py_RETURN_NONE; + + if (!HasOverlappedIoCompleted(&self->overlapped)) { + Py_BEGIN_ALLOW_THREADS + if (Py_CancelIoEx) + ret = Py_CancelIoEx(self->handle, &self->overlapped); + else + ret = CancelIo(self->handle); + Py_END_ALLOW_THREADS + } + + /* CancelIoEx returns ERROR_NOT_FOUND if the I/O completed in-between */ + if (!ret && GetLastError() != ERROR_NOT_FOUND) + return SetFromWindowsErr(0); + Py_RETURN_NONE; +} + +PyDoc_STRVAR( + Overlapped_getresult_doc, + "getresult(wait=False) -> result\n\n" + "Retrieve result of operation. If wait is true then it blocks\n" + "until the operation is finished. If wait is false and the\n" + "operation is still pending then an error is raised."); + +static PyObject * +Overlapped_getresult(OverlappedObject *self, PyObject *args) +{ + BOOL wait = FALSE; + DWORD transferred = 0; + BOOL ret; + DWORD err; + + if (!PyArg_ParseTuple(args, "|" F_BOOL, &wait)) + return NULL; + + if (self->type == TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation not yet attempted"); + return NULL; + } + + if (self->type == TYPE_NOT_STARTED) { + PyErr_SetString(PyExc_ValueError, "operation failed to start"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + ret = GetOverlappedResult(self->handle, &self->overlapped, &transferred, + wait); + Py_END_ALLOW_THREADS + + self->error = err = ret ? ERROR_SUCCESS : GetLastError(); + switch (err) { + case ERROR_SUCCESS: + case ERROR_MORE_DATA: + break; + case ERROR_BROKEN_PIPE: + if ((self->type == TYPE_READ || self->type == TYPE_ACCEPT) && self->read_buffer != NULL) + break; + /* fall through */ + default: + return SetFromWindowsErr(err); + } + + switch (self->type) { + case TYPE_READ: + assert(PyBytes_CheckExact(self->read_buffer)); + if (transferred != PyBytes_GET_SIZE(self->read_buffer) && + _PyBytes_Resize(&self->read_buffer, transferred)) + return NULL; + Py_INCREF(self->read_buffer); + return self->read_buffer; + default: + return PyLong_FromUnsignedLong((unsigned long) transferred); + } +} + +PyDoc_STRVAR( + Overlapped_ReadFile_doc, + "ReadFile(handle, size) -> Overlapped[message]\n\n" + "Start overlapped read"); + +static PyObject * +Overlapped_ReadFile(OverlappedObject *self, PyObject *args) +{ + HANDLE handle; + DWORD size; + DWORD nread; + PyObject *buf; + BOOL ret; + DWORD err; + + if (!PyArg_ParseTuple(args, F_HANDLE F_DWORD, &handle, &size)) + return NULL; + + if (self->type != TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation already attempted"); + return NULL; + } + +#if SIZEOF_SIZE_T <= SIZEOF_LONG + size = Py_MIN(size, (DWORD)PY_SSIZE_T_MAX); +#endif + buf = PyBytes_FromStringAndSize(NULL, Py_MAX(size, 1)); + if (buf == NULL) + return NULL; + + self->type = TYPE_READ; + self->handle = handle; + self->read_buffer = buf; + + Py_BEGIN_ALLOW_THREADS + ret = ReadFile(handle, PyBytes_AS_STRING(buf), size, &nread, + &self->overlapped); + Py_END_ALLOW_THREADS + + self->error = err = ret ? ERROR_SUCCESS : GetLastError(); + switch (err) { + case ERROR_BROKEN_PIPE: + mark_as_completed(&self->overlapped); + return SetFromWindowsErr(err); + case ERROR_SUCCESS: + case ERROR_MORE_DATA: + case ERROR_IO_PENDING: + Py_RETURN_NONE; + default: + self->type = TYPE_NOT_STARTED; + return SetFromWindowsErr(err); + } +} + +PyDoc_STRVAR( + Overlapped_WSARecv_doc, + "RecvFile(handle, size, flags) -> Overlapped[message]\n\n" + "Start overlapped receive"); + +static PyObject * +Overlapped_WSARecv(OverlappedObject *self, PyObject *args) +{ + HANDLE handle; + DWORD size; + DWORD flags = 0; + DWORD nread; + PyObject *buf; + WSABUF wsabuf; + int ret; + DWORD err; + + if (!PyArg_ParseTuple(args, F_HANDLE F_DWORD "|" F_DWORD, + &handle, &size, &flags)) + return NULL; + + if (self->type != TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation already attempted"); + return NULL; + } + +#if SIZEOF_SIZE_T <= SIZEOF_LONG + size = Py_MIN(size, (DWORD)PY_SSIZE_T_MAX); +#endif + buf = PyBytes_FromStringAndSize(NULL, Py_MAX(size, 1)); + if (buf == NULL) + return NULL; + + self->type = TYPE_READ; + self->handle = handle; + self->read_buffer = buf; + wsabuf.len = size; + wsabuf.buf = PyBytes_AS_STRING(buf); + + Py_BEGIN_ALLOW_THREADS + ret = WSARecv((SOCKET)handle, &wsabuf, 1, &nread, &flags, + &self->overlapped, NULL); + Py_END_ALLOW_THREADS + + self->error = err = (ret < 0 ? WSAGetLastError() : ERROR_SUCCESS); + switch (err) { + case ERROR_BROKEN_PIPE: + mark_as_completed(&self->overlapped); + return SetFromWindowsErr(err); + case ERROR_SUCCESS: + case ERROR_MORE_DATA: + case ERROR_IO_PENDING: + Py_RETURN_NONE; + default: + self->type = TYPE_NOT_STARTED; + return SetFromWindowsErr(err); + } +} + +PyDoc_STRVAR( + Overlapped_WriteFile_doc, + "WriteFile(handle, buf) -> Overlapped[bytes_transferred]\n\n" + "Start overlapped write"); + +static PyObject * +Overlapped_WriteFile(OverlappedObject *self, PyObject *args) +{ + HANDLE handle; + PyObject *bufobj; + DWORD written; + BOOL ret; + DWORD err; + + if (!PyArg_ParseTuple(args, F_HANDLE "O", &handle, &bufobj)) + return NULL; + + if (self->type != TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation already attempted"); + return NULL; + } + + if (!PyArg_Parse(bufobj, "y*", &self->write_buffer)) + return NULL; + +#if SIZEOF_SIZE_T > SIZEOF_LONG + if (self->write_buffer.len > (Py_ssize_t)ULONG_MAX) { + PyBuffer_Release(&self->write_buffer); + PyErr_SetString(PyExc_ValueError, "buffer to large"); + return NULL; + } +#endif + + self->type = TYPE_WRITE; + self->handle = handle; + + Py_BEGIN_ALLOW_THREADS + ret = WriteFile(handle, self->write_buffer.buf, + (DWORD)self->write_buffer.len, + &written, &self->overlapped); + Py_END_ALLOW_THREADS + + self->error = err = ret ? ERROR_SUCCESS : GetLastError(); + switch (err) { + case ERROR_SUCCESS: + case ERROR_IO_PENDING: + Py_RETURN_NONE; + default: + self->type = TYPE_NOT_STARTED; + return SetFromWindowsErr(err); + } +} + +PyDoc_STRVAR( + Overlapped_WSASend_doc, + "WSASend(handle, buf, flags) -> Overlapped[bytes_transferred]\n\n" + "Start overlapped send"); + +static PyObject * +Overlapped_WSASend(OverlappedObject *self, PyObject *args) +{ + HANDLE handle; + PyObject *bufobj; + DWORD flags; + DWORD written; + WSABUF wsabuf; + int ret; + DWORD err; + + if (!PyArg_ParseTuple(args, F_HANDLE "O" F_DWORD, + &handle, &bufobj, &flags)) + return NULL; + + if (self->type != TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation already attempted"); + return NULL; + } + + if (!PyArg_Parse(bufobj, "y*", &self->write_buffer)) + return NULL; + +#if SIZEOF_SIZE_T > SIZEOF_LONG + if (self->write_buffer.len > (Py_ssize_t)ULONG_MAX) { + PyBuffer_Release(&self->write_buffer); + PyErr_SetString(PyExc_ValueError, "buffer to large"); + return NULL; + } +#endif + + self->type = TYPE_WRITE; + self->handle = handle; + wsabuf.len = (DWORD)self->write_buffer.len; + wsabuf.buf = self->write_buffer.buf; + + Py_BEGIN_ALLOW_THREADS + ret = WSASend((SOCKET)handle, &wsabuf, 1, &written, flags, + &self->overlapped, NULL); + Py_END_ALLOW_THREADS + + self->error = err = (ret < 0 ? WSAGetLastError() : ERROR_SUCCESS); + switch (err) { + case ERROR_SUCCESS: + case ERROR_IO_PENDING: + Py_RETURN_NONE; + default: + self->type = TYPE_NOT_STARTED; + return SetFromWindowsErr(err); + } +} + +PyDoc_STRVAR( + Overlapped_AcceptEx_doc, + "AcceptEx(listen_handle, accept_handle) -> Overlapped[address_as_bytes]\n\n" + "Start overlapped wait for client to connect"); + +static PyObject * +Overlapped_AcceptEx(OverlappedObject *self, PyObject *args) +{ + SOCKET ListenSocket; + SOCKET AcceptSocket; + DWORD BytesReceived; + DWORD size; + PyObject *buf; + BOOL ret; + DWORD err; + + if (!PyArg_ParseTuple(args, F_HANDLE F_HANDLE, + &ListenSocket, &AcceptSocket)) + return NULL; + + if (self->type != TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation already attempted"); + return NULL; + } + + size = sizeof(struct sockaddr_in6) + 16; + buf = PyBytes_FromStringAndSize(NULL, size*2); + if (!buf) + return NULL; + + self->type = TYPE_ACCEPT; + self->handle = (HANDLE)ListenSocket; + self->read_buffer = buf; + + Py_BEGIN_ALLOW_THREADS + ret = Py_AcceptEx(ListenSocket, AcceptSocket, PyBytes_AS_STRING(buf), + 0, size, size, &BytesReceived, &self->overlapped); + Py_END_ALLOW_THREADS + + self->error = err = ret ? ERROR_SUCCESS : WSAGetLastError(); + switch (err) { + case ERROR_SUCCESS: + case ERROR_IO_PENDING: + Py_RETURN_NONE; + default: + self->type = TYPE_NOT_STARTED; + return SetFromWindowsErr(err); + } +} + + +static int +parse_address(PyObject *obj, SOCKADDR *Address, int Length) +{ + char *Host; + unsigned short Port; + unsigned long FlowInfo; + unsigned long ScopeId; + + memset(Address, 0, Length); + + if (PyArg_ParseTuple(obj, "sH", &Host, &Port)) + { + Address->sa_family = AF_INET; + if (WSAStringToAddressA(Host, AF_INET, NULL, Address, &Length) < 0) { + SetFromWindowsErr(WSAGetLastError()); + return -1; + } + ((SOCKADDR_IN*)Address)->sin_port = htons(Port); + return Length; + } + else if (PyArg_ParseTuple(obj, "sHkk", &Host, &Port, &FlowInfo, &ScopeId)) + { + PyErr_Clear(); + Address->sa_family = AF_INET6; + if (WSAStringToAddressA(Host, AF_INET6, NULL, Address, &Length) < 0) { + SetFromWindowsErr(WSAGetLastError()); + return -1; + } + ((SOCKADDR_IN6*)Address)->sin6_port = htons(Port); + ((SOCKADDR_IN6*)Address)->sin6_flowinfo = FlowInfo; + ((SOCKADDR_IN6*)Address)->sin6_scope_id = ScopeId; + return Length; + } + + return -1; +} + + +PyDoc_STRVAR( + Overlapped_ConnectEx_doc, + "ConnectEx(client_handle, address_as_bytes) -> Overlapped[None]\n\n" + "Start overlapped connect. client_handle should be unbound."); + +static PyObject * +Overlapped_ConnectEx(OverlappedObject *self, PyObject *args) +{ + SOCKET ConnectSocket; + PyObject *AddressObj; + char AddressBuf[sizeof(struct sockaddr_in6)]; + SOCKADDR *Address = (SOCKADDR*)AddressBuf; + int Length; + BOOL ret; + DWORD err; + + if (!PyArg_ParseTuple(args, F_HANDLE "O", &ConnectSocket, &AddressObj)) + return NULL; + + if (self->type != TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation already attempted"); + return NULL; + } + + Length = sizeof(AddressBuf); + Length = parse_address(AddressObj, Address, Length); + if (Length < 0) + return NULL; + + self->type = TYPE_CONNECT; + self->handle = (HANDLE)ConnectSocket; + + Py_BEGIN_ALLOW_THREADS + ret = Py_ConnectEx(ConnectSocket, Address, Length, + NULL, 0, NULL, &self->overlapped); + Py_END_ALLOW_THREADS + + self->error = err = ret ? ERROR_SUCCESS : WSAGetLastError(); + switch (err) { + case ERROR_SUCCESS: + case ERROR_IO_PENDING: + Py_RETURN_NONE; + default: + self->type = TYPE_NOT_STARTED; + return SetFromWindowsErr(err); + } +} + +PyDoc_STRVAR( + Overlapped_DisconnectEx_doc, + "DisconnectEx(handle, flags) -> Overlapped[None]\n\n" + "Start overlapped connect. client_handle should be unbound."); + +static PyObject * +Overlapped_DisconnectEx(OverlappedObject *self, PyObject *args) +{ + SOCKET Socket; + DWORD flags; + BOOL ret; + DWORD err; + + if (!PyArg_ParseTuple(args, F_HANDLE F_DWORD, &Socket, &flags)) + return NULL; + + if (self->type != TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation already attempted"); + return NULL; + } + + self->type = TYPE_DISCONNECT; + self->handle = (HANDLE)Socket; + + Py_BEGIN_ALLOW_THREADS + ret = Py_DisconnectEx(Socket, &self->overlapped, flags, 0); + Py_END_ALLOW_THREADS + + self->error = err = ret ? ERROR_SUCCESS : WSAGetLastError(); + switch (err) { + case ERROR_SUCCESS: + case ERROR_IO_PENDING: + Py_RETURN_NONE; + default: + self->type = TYPE_NOT_STARTED; + return SetFromWindowsErr(err); + } +} + +PyDoc_STRVAR( + Overlapped_ConnectNamedPipe_doc, + "ConnectNamedPipe(handle) -> Overlapped[None]\n\n" + "Start overlapped wait for a client to connect."); + +static PyObject * +Overlapped_ConnectNamedPipe(OverlappedObject *self, PyObject *args) +{ + HANDLE Pipe; + BOOL ret; + DWORD err; + + if (!PyArg_ParseTuple(args, F_HANDLE, &Pipe)) + return NULL; + + if (self->type != TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation already attempted"); + return NULL; + } + + self->type = TYPE_CONNECT_NAMED_PIPE; + self->handle = Pipe; + + Py_BEGIN_ALLOW_THREADS + ret = ConnectNamedPipe(Pipe, &self->overlapped); + Py_END_ALLOW_THREADS + + self->error = err = ret ? ERROR_SUCCESS : GetLastError(); + switch (err) { + case ERROR_PIPE_CONNECTED: + mark_as_completed(&self->overlapped); + Py_RETURN_TRUE; + case ERROR_SUCCESS: + case ERROR_IO_PENDING: + Py_RETURN_FALSE; + default: + self->type = TYPE_NOT_STARTED; + return SetFromWindowsErr(err); + } +} + +PyDoc_STRVAR( + ConnectPipe_doc, + "ConnectPipe(addr) -> pipe_handle\n\n" + "Connect to the pipe for asynchronous I/O (overlapped)."); + +static PyObject * +ConnectPipe(OverlappedObject *self, PyObject *args) +{ + PyObject *AddressObj; + wchar_t *Address; + HANDLE PipeHandle; + + if (!PyArg_ParseTuple(args, "U", &AddressObj)) + return NULL; + + Address = PyUnicode_AsWideCharString(AddressObj, NULL); + if (Address == NULL) + return NULL; + + Py_BEGIN_ALLOW_THREADS + PipeHandle = CreateFileW(Address, + GENERIC_READ | GENERIC_WRITE, + 0, NULL, OPEN_EXISTING, + FILE_FLAG_OVERLAPPED, NULL); + Py_END_ALLOW_THREADS + + PyMem_Free(Address); + if (PipeHandle == INVALID_HANDLE_VALUE) + return SetFromWindowsErr(0); + return Py_BuildValue(F_HANDLE, PipeHandle); +} + +static PyObject* +Overlapped_getaddress(OverlappedObject *self) +{ + return PyLong_FromVoidPtr(&self->overlapped); +} + +static PyObject* +Overlapped_getpending(OverlappedObject *self) +{ + return PyBool_FromLong(!HasOverlappedIoCompleted(&self->overlapped) && + self->type != TYPE_NOT_STARTED); +} + +static PyMethodDef Overlapped_methods[] = { + {"getresult", (PyCFunction) Overlapped_getresult, + METH_VARARGS, Overlapped_getresult_doc}, + {"cancel", (PyCFunction) Overlapped_cancel, + METH_NOARGS, Overlapped_cancel_doc}, + {"ReadFile", (PyCFunction) Overlapped_ReadFile, + METH_VARARGS, Overlapped_ReadFile_doc}, + {"WSARecv", (PyCFunction) Overlapped_WSARecv, + METH_VARARGS, Overlapped_WSARecv_doc}, + {"WriteFile", (PyCFunction) Overlapped_WriteFile, + METH_VARARGS, Overlapped_WriteFile_doc}, + {"WSASend", (PyCFunction) Overlapped_WSASend, + METH_VARARGS, Overlapped_WSASend_doc}, + {"AcceptEx", (PyCFunction) Overlapped_AcceptEx, + METH_VARARGS, Overlapped_AcceptEx_doc}, + {"ConnectEx", (PyCFunction) Overlapped_ConnectEx, + METH_VARARGS, Overlapped_ConnectEx_doc}, + {"DisconnectEx", (PyCFunction) Overlapped_DisconnectEx, + METH_VARARGS, Overlapped_DisconnectEx_doc}, + {"ConnectNamedPipe", (PyCFunction) Overlapped_ConnectNamedPipe, + METH_VARARGS, Overlapped_ConnectNamedPipe_doc}, + {NULL} +}; + +static PyMemberDef Overlapped_members[] = { + {"error", T_ULONG, + offsetof(OverlappedObject, error), + READONLY, "Error from last operation"}, + {"event", T_HANDLE, + offsetof(OverlappedObject, overlapped) + offsetof(OVERLAPPED, hEvent), + READONLY, "Overlapped event handle"}, + {NULL} +}; + +static PyGetSetDef Overlapped_getsets[] = { + {"address", (getter)Overlapped_getaddress, NULL, + "Address of overlapped structure"}, + {"pending", (getter)Overlapped_getpending, NULL, + "Whether the operation is pending"}, + {NULL}, +}; + +PyTypeObject OverlappedType = { + PyVarObject_HEAD_INIT(NULL, 0) + /* tp_name */ "_overlapped.Overlapped", + /* tp_basicsize */ sizeof(OverlappedObject), + /* tp_itemsize */ 0, + /* tp_dealloc */ (destructor) Overlapped_dealloc, + /* tp_print */ 0, + /* tp_getattr */ 0, + /* tp_setattr */ 0, + /* tp_reserved */ 0, + /* tp_repr */ 0, + /* tp_as_number */ 0, + /* tp_as_sequence */ 0, + /* tp_as_mapping */ 0, + /* tp_hash */ 0, + /* tp_call */ 0, + /* tp_str */ 0, + /* tp_getattro */ 0, + /* tp_setattro */ 0, + /* tp_as_buffer */ 0, + /* tp_flags */ Py_TPFLAGS_DEFAULT, + /* tp_doc */ "OVERLAPPED structure wrapper", + /* tp_traverse */ 0, + /* tp_clear */ 0, + /* tp_richcompare */ 0, + /* tp_weaklistoffset */ 0, + /* tp_iter */ 0, + /* tp_iternext */ 0, + /* tp_methods */ Overlapped_methods, + /* tp_members */ Overlapped_members, + /* tp_getset */ Overlapped_getsets, + /* tp_base */ 0, + /* tp_dict */ 0, + /* tp_descr_get */ 0, + /* tp_descr_set */ 0, + /* tp_dictoffset */ 0, + /* tp_init */ 0, + /* tp_alloc */ 0, + /* tp_new */ Overlapped_new, +}; + +static PyMethodDef overlapped_functions[] = { + {"CreateIoCompletionPort", overlapped_CreateIoCompletionPort, + METH_VARARGS, CreateIoCompletionPort_doc}, + {"GetQueuedCompletionStatus", overlapped_GetQueuedCompletionStatus, + METH_VARARGS, GetQueuedCompletionStatus_doc}, + {"PostQueuedCompletionStatus", overlapped_PostQueuedCompletionStatus, + METH_VARARGS, PostQueuedCompletionStatus_doc}, + {"FormatMessage", overlapped_FormatMessage, + METH_VARARGS, FormatMessage_doc}, + {"BindLocal", overlapped_BindLocal, + METH_VARARGS, BindLocal_doc}, + {"RegisterWaitWithQueue", overlapped_RegisterWaitWithQueue, + METH_VARARGS, RegisterWaitWithQueue_doc}, + {"UnregisterWait", overlapped_UnregisterWait, + METH_VARARGS, UnregisterWait_doc}, + {"UnregisterWaitEx", overlapped_UnregisterWaitEx, + METH_VARARGS, UnregisterWaitEx_doc}, + {"CreateEvent", overlapped_CreateEvent, + METH_VARARGS, CreateEvent_doc}, + {"SetEvent", overlapped_SetEvent, + METH_VARARGS, SetEvent_doc}, + {"ResetEvent", overlapped_ResetEvent, + METH_VARARGS, ResetEvent_doc}, + {"ConnectPipe", + (PyCFunction) ConnectPipe, + METH_VARARGS, ConnectPipe_doc}, + {NULL} +}; + +static struct PyModuleDef overlapped_module = { + PyModuleDef_HEAD_INIT, + "_overlapped", + NULL, + -1, + overlapped_functions, + NULL, + NULL, + NULL, + NULL +}; + +#define WINAPI_CONSTANT(fmt, con) \ + PyDict_SetItemString(d, #con, Py_BuildValue(fmt, con)) + +PyMODINIT_FUNC +PyInit__overlapped(void) +{ + PyObject *m, *d; + + /* Ensure WSAStartup() called before initializing function pointers */ + m = PyImport_ImportModule("_socket"); + if (!m) + return NULL; + Py_DECREF(m); + + if (initialize_function_pointers() < 0) + return NULL; + + if (PyType_Ready(&OverlappedType) < 0) + return NULL; + + m = PyModule_Create(&overlapped_module); + if (PyModule_AddObject(m, "Overlapped", (PyObject *)&OverlappedType) < 0) + return NULL; + + d = PyModule_GetDict(m); + + WINAPI_CONSTANT(F_DWORD, ERROR_IO_PENDING); + WINAPI_CONSTANT(F_DWORD, ERROR_NETNAME_DELETED); + WINAPI_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); + WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); + WINAPI_CONSTANT(F_DWORD, INFINITE); + WINAPI_CONSTANT(F_HANDLE, INVALID_HANDLE_VALUE); + WINAPI_CONSTANT(F_HANDLE, NULL); + WINAPI_CONSTANT(F_DWORD, SO_UPDATE_ACCEPT_CONTEXT); + WINAPI_CONSTANT(F_DWORD, SO_UPDATE_CONNECT_CONTEXT); + WINAPI_CONSTANT(F_DWORD, TF_REUSE_SOCKET); + + return m; +} diff --git a/thirdparty/asyncio/pypi.bat b/thirdparty/asyncio/pypi.bat new file mode 100644 index 0000000..5218ace --- /dev/null +++ b/thirdparty/asyncio/pypi.bat @@ -0,0 +1 @@ +c:\Python33\python.exe setup.py bdist_wheel upload diff --git a/thirdparty/asyncio/release.py b/thirdparty/asyncio/release.py new file mode 100644 index 0000000..0cb80a3 --- /dev/null +++ b/thirdparty/asyncio/release.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +""" +Script to upload 32 bits and 64 bits wheel packages for Python 3.3 on Windows. + +Usage: "python release.py HG_TAG" where HG_TAG is a Mercurial tag, usually +a version number like "3.4.2". + +Requirements: + +- Python 3.3 and newer requires the Windows SDK 7.1 to build wheel packages +- Python 2.7 requires the Windows SDK 7.0 +- the aiotest module is required to run aiotest tests +""" +import contextlib +import optparse +import os +import platform +import re +import shutil +import subprocess +import sys +import tempfile + +PROJECT = 'asyncio' +DEBUG_ENV_VAR = 'PYTHONASYNCIODEBUG' +PYTHON_VERSIONS = ( + (3, 3), +) +PY3 = (sys.version_info >= (3,)) +HG = 'hg' +SDK_ROOT = r"C:\Program Files\Microsoft SDKs\Windows" +BATCH_FAIL_ON_ERROR = "@IF %errorlevel% neq 0 exit /b %errorlevel%" +WINDOWS = (sys.platform == 'win32') + + +def get_architecture_bits(): + arch = platform.architecture()[0] + return int(arch[:2]) + + +class PythonVersion: + def __init__(self, major, minor, bits): + self.major = major + self.minor = minor + self.bits = bits + self._executable = None + + @staticmethod + def running(): + bits = get_architecture_bits() + pyver = PythonVersion(sys.version_info.major, + sys.version_info.minor, + bits) + pyver._executable = sys.executable + return pyver + + def _get_executable_windows(self, app): + if self.bits == 32: + executable = 'c:\\Python%s%s_32bit\\python.exe' + else: + executable = 'c:\\Python%s%s\\python.exe' + executable = executable % (self.major, self.minor) + if not os.path.exists(executable): + print("Unable to find python %s" % self) + print("%s does not exists" % executable) + sys.exit(1) + return executable + + def _get_executable_unix(self, app): + return 'python%s.%s' % (self.major, self.minor) + + def get_executable(self, app): + if self._executable: + return self._executable + + if WINDOWS: + executable = self._get_executable_windows(app) + else: + executable = self._get_executable_unix(app) + + code = ( + 'import platform, sys; ' + 'print("{ver.major}.{ver.minor} {bits}".format(' + 'ver=sys.version_info, ' + 'bits=platform.architecture()[0]))' + ) + try: + exitcode, stdout = app.get_output(executable, '-c', code, + ignore_stderr=True) + except OSError as exc: + print("Error while checking %s:" % self) + print(str(exc)) + print("Executable: %s" % executable) + sys.exit(1) + else: + stdout = stdout.rstrip() + expected = "%s.%s %sbit" % (self.major, self.minor, self.bits) + if stdout != expected: + print("Python version or architecture doesn't match") + print("got %r, expected %r" % (stdout, expected)) + print("Executable: %s" % executable) + sys.exit(1) + + self._executable = executable + return executable + + def __str__(self): + return 'Python %s.%s (%s bits)' % (self.major, self.minor, self.bits) + + +class Release(object): + def __init__(self): + root = os.path.dirname(__file__) + self.root = os.path.realpath(root) + # Set these attributes to True to run also register sdist upload + self.wheel = False + self.test = False + self.register = False + self.sdist = False + self.aiotest = False + self.verbose = False + self.upload = False + # Release mode: enable more tests + self.release = False + self.python_versions = [] + if WINDOWS: + supported_archs = (32, 64) + else: + bits = get_architecture_bits() + supported_archs = (bits,) + for major, minor in PYTHON_VERSIONS: + for bits in supported_archs: + pyver = PythonVersion(major, minor, bits) + self.python_versions.append(pyver) + + @contextlib.contextmanager + def _popen(self, args, **kw): + verbose = kw.pop('verbose', True) + if self.verbose and verbose: + print('+ ' + ' '.join(args)) + if PY3: + kw['universal_newlines'] = True + proc = subprocess.Popen(args, **kw) + try: + yield proc + except: + proc.kill() + proc.wait() + raise + + def get_output(self, *args, **kw): + kw['stdout'] = subprocess.PIPE + ignore_stderr = kw.pop('ignore_stderr', False) + if ignore_stderr: + devnull = open(os.path.devnull, 'wb') + kw['stderr'] = devnull + else: + kw['stderr'] = subprocess.STDOUT + try: + with self._popen(args, **kw) as proc: + stdout, stderr = proc.communicate() + return proc.returncode, stdout + finally: + if ignore_stderr: + devnull.close() + + def check_output(self, *args, **kw): + exitcode, output = self.get_output(*args, **kw) + if exitcode: + sys.stdout.write(output) + sys.stdout.flush() + sys.exit(1) + return output + + def run_command(self, *args, **kw): + with self._popen(args, **kw) as proc: + exitcode = proc.wait() + if exitcode: + sys.exit(exitcode) + + def get_local_changes(self): + status = self.check_output(HG, 'status') + return [line for line in status.splitlines() + if not line.startswith("?")] + + def remove_directory(self, name): + path = os.path.join(self.root, name) + if os.path.exists(path): + if self.verbose: + print("Remove directory: %s" % name) + shutil.rmtree(path) + + def remove_file(self, name): + path = os.path.join(self.root, name) + if os.path.exists(path): + if self.verbose: + print("Remove file: %s" % name) + os.unlink(path) + + def windows_sdk_setenv(self, pyver): + if (pyver.major, pyver.minor) >= (3, 3): + path = "v7.1" + sdkver = (7, 1) + else: + path = "v7.0" + sdkver = (7, 0) + setenv = os.path.join(SDK_ROOT, path, 'Bin', 'SetEnv.cmd') + if not os.path.exists(setenv): + print("Unable to find Windows SDK %s.%s for %s" + % (sdkver[0], sdkver[1], pyver)) + print("Please download and install it") + print("%s does not exists" % setenv) + sys.exit(1) + if pyver.bits == 64: + arch = '/x64' + else: + arch = '/x86' + cmd = ["CALL", setenv, "/release", arch] + return (cmd, sdkver) + + def quote(self, arg): + if not re.search("[ '\"]", arg): + return arg + # FIXME: should we escape "? + return '"%s"' % arg + + def quote_args(self, args): + return ' '.join(self.quote(arg) for arg in args) + + def cleanup(self): + if self.verbose: + print("Cleanup") + self.remove_directory('build') + self.remove_directory('dist') + self.remove_file('_overlapped.pyd') + self.remove_file(os.path.join(PROJECT, '_overlapped.pyd')) + + def sdist_upload(self): + self.cleanup() + self.run_command(sys.executable, 'setup.py', 'sdist', 'upload') + + def build_inplace(self, pyver): + print("Build for %s" % pyver) + self.build(pyver, 'build') + + if WINDOWS: + if pyver.bits == 64: + arch = 'win-amd64' + else: + arch = 'win32' + build_dir = 'lib.%s-%s.%s' % (arch, pyver.major, pyver.minor) + src = os.path.join(self.root, 'build', build_dir, + PROJECT, '_overlapped.pyd') + dst = os.path.join(self.root, PROJECT, '_overlapped.pyd') + shutil.copyfile(src, dst) + + def runtests(self, pyver): + print("Run tests on %s" % pyver) + + if WINDOWS and not self.options.no_compile: + self.build_inplace(pyver) + + release_env = dict(os.environ) + release_env.pop(DEBUG_ENV_VAR, None) + + dbg_env = dict(os.environ) + dbg_env[DEBUG_ENV_VAR] = '1' + + python = pyver.get_executable(self) + args = (python, 'runtests.py', '-r') + + if self.release: + print("Run runtests.py in release mode on %s" % pyver) + self.run_command(*args, env=release_env) + + print("Run runtests.py in debug mode on %s" % pyver) + self.run_command(*args, env=dbg_env) + + if self.aiotest: + args = (python, 'run_aiotest.py') + + if self.release: + print("Run aiotest in release mode on %s" % pyver) + self.run_command(*args, env=release_env) + + print("Run aiotest in debug mode on %s" % pyver) + self.run_command(*args, env=dbg_env) + print("") + + def _build_windows(self, pyver, cmd): + setenv, sdkver = self.windows_sdk_setenv(pyver) + + temp = tempfile.NamedTemporaryFile(mode="w", suffix=".bat", + delete=False) + with temp: + temp.write("SETLOCAL EnableDelayedExpansion\n") + temp.write(self.quote_args(setenv) + "\n") + temp.write(BATCH_FAIL_ON_ERROR + "\n") + # Restore console colors: lightgrey on black + temp.write("COLOR 07\n") + temp.write("\n") + temp.write("SET DISTUTILS_USE_SDK=1\n") + temp.write("SET MSSDK=1\n") + temp.write("CD %s\n" % self.quote(self.root)) + temp.write(self.quote_args(cmd) + "\n") + temp.write(BATCH_FAIL_ON_ERROR + "\n") + + try: + if self.verbose: + print("Setup Windows SDK %s.%s" % sdkver) + print("+ " + ' '.join(cmd)) + # SDK 7.1 uses the COLOR command which makes SetEnv.cmd failing + # if the stdout is not a TTY (if we redirect stdout into a file) + if self.verbose or sdkver >= (7, 1): + self.run_command(temp.name, verbose=False) + else: + self.check_output(temp.name, verbose=False) + finally: + os.unlink(temp.name) + + def _build_unix(self, pyver, cmd): + self.check_output(*cmd) + + def build(self, pyver, *cmds): + self.cleanup() + + python = pyver.get_executable(self) + cmd = [python, 'setup.py'] + list(cmds) + + if WINDOWS: + self._build_windows(pyver, cmd) + else: + self._build_unix(pyver, cmd) + + def test_wheel(self, pyver): + print("Test building wheel package for %s" % pyver) + self.build(pyver, 'bdist_wheel') + + def publish_wheel(self, pyver): + print("Build and publish wheel package for %s" % pyver) + self.build(pyver, 'bdist_wheel', 'upload') + + def parse_options(self): + parser = optparse.OptionParser( + description="Run all unittests.", + usage="%prog [options] command") + parser.add_option( + '-v', '--verbose', action="store_true", dest='verbose', + default=0, help='verbose') + parser.add_option( + '-t', '--tag', type="str", + help='Mercurial tag or revision, required to release') + parser.add_option( + '-p', '--python', type="str", + help='Only build/test one specific Python version, ex: "2.7:32"') + parser.add_option( + '-C', "--no-compile", action="store_true", + help="Don't compile the module, this options implies --running", + default=False) + parser.add_option( + '-r', "--running", action="store_true", + help='Only use the running Python version', + default=False) + parser.add_option( + '--ignore', action="store_true", + help='Ignore local changes', + default=False) + self.options, args = parser.parse_args() + if len(args) == 1: + command = args[0] + else: + command = None + + if self.options.no_compile: + self.options.running = True + + if command == 'clean': + self.options.verbose = True + elif command == 'build': + self.options.running = True + elif command == 'test_wheel': + self.wheel = True + elif command == 'test': + self.test = True + elif command == 'release': + if not self.options.tag: + print("The release command requires the --tag option") + sys.exit(1) + + self.release = True + self.wheel = True + self.test = True + self.upload = True + else: + if command: + print("Invalid command: %s" % command) + else: + parser.print_help() + print("") + + print("Available commands:") + print("- build: build asyncio in place, imply --running") + print("- test: run tests") + print("- test_wheel: test building wheel packages") + print("- release: run tests and publish wheel packages,") + print(" require the --tag option") + print("- clean: cleanup the project") + sys.exit(1) + + if self.options.python and self.options.running: + print("--python and --running options are exclusive") + sys.exit(1) + + python = self.options.python + if python: + match = re.match("^([23])\.([0-9])/(32|64)$", python) + if not match: + print("Invalid Python version: %s" % python) + print('Format of a Python version: "x.y/bits"') + print("Example: 2.7/32") + sys.exit(1) + major = int(match.group(1)) + minor = int(match.group(2)) + bits = int(match.group(3)) + self.python_versions = [PythonVersion(major, minor, bits)] + + if self.options.running: + self.python_versions = [PythonVersion.running()] + + self.verbose = self.options.verbose + self.command = command + + def main(self): + self.parse_options() + + print("Directory: %s" % self.root) + os.chdir(self.root) + + if self.command == "clean": + self.cleanup() + sys.exit(1) + + if self.command == "build": + if len(self.python_versions) != 1: + print("build command requires one specific Python version") + print("Use the --python command line option") + sys.exit(1) + pyver = self.python_versions[0] + self.build_inplace(pyver) + + if (self.register or self.upload) and (not self.options.ignore): + lines = self.get_local_changes() + else: + lines = () + if lines: + print("ERROR: Found local changes") + for line in lines: + print(line) + print("") + print("Revert local changes") + print("or use the --ignore command line option") + sys.exit(1) + + hg_tag = self.options.tag + if hg_tag: + print("Update repository to revision %s" % hg_tag) + self.check_output(HG, 'update', hg_tag) + + hg_rev = self.check_output(HG, 'id').rstrip() + + if self.wheel: + for pyver in self.python_versions: + self.test_wheel(pyver) + + if self.test: + for pyver in self.python_versions: + self.runtests(pyver) + + if self.register: + self.run_command(sys.executable, 'setup.py', 'register') + + if self.sdist: + self.sdist_upload() + + if self.upload: + for pyver in self.python_versions: + self.publish_wheel(pyver) + + hg_rev2 = self.check_output(HG, 'id').rstrip() + if hg_rev != hg_rev2: + print("ERROR: The Mercurial revision changed") + print("Before: %s" % hg_rev) + print("After: %s" % hg_rev2) + sys.exit(1) + + print("") + print("Mercurial revision: %s" % hg_rev) + if self.command == 'build': + print("Inplace compilation done") + if self.wheel: + print("Compilation of wheel packages succeeded") + if self.test: + print("Tests succeeded") + if self.register: + print("Project registered on the Python cheeseshop (PyPI)") + if self.sdist: + print("Project source code uploaded to the Python " + "cheeseshop (PyPI)") + if self.upload: + print("Wheel packages uploaded to the Python cheeseshop (PyPI)") + for pyver in self.python_versions: + print("- %s" % pyver) + + +if __name__ == "__main__": + Release().main() diff --git a/thirdparty/asyncio/run_aiotest.py b/thirdparty/asyncio/run_aiotest.py new file mode 100644 index 0000000..8d6fa29 --- /dev/null +++ b/thirdparty/asyncio/run_aiotest.py @@ -0,0 +1,14 @@ +import aiotest.run +import asyncio +import sys +if sys.platform == 'win32': + from asyncio.windows_utils import socketpair +else: + from socket import socketpair + +config = aiotest.TestConfig() +config.asyncio = asyncio +config.socketpair = socketpair +config.new_event_pool_policy = asyncio.DefaultEventLoopPolicy +config.call_soon_check_closed = True +aiotest.run.main(config) diff --git a/thirdparty/asyncio/runtests.py b/thirdparty/asyncio/runtests.py new file mode 100644 index 0000000..c407462 --- /dev/null +++ b/thirdparty/asyncio/runtests.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +"""Run asyncio unittests. + +Usage: + python3 runtests.py [flags] [pattern] ... + +Patterns are matched against the fully qualified name of the test, +including package, module, class and method, +e.g. 'tests.test_events.PolicyTests.testPolicy'. + +For full help, try --help. + +runtests.py --coverage is equivalent of: + + $(COVERAGE) run --branch runtests.py -v + $(COVERAGE) html $(list of files) + $(COVERAGE) report -m $(list of files) + +""" + +# Originally written by Beech Horn (for NDB). + +import argparse +import gc +import logging +import os +import random +import re +import sys +import unittest +import textwrap +import warnings +import importlib.machinery +try: + import coverage +except ImportError: + coverage = None + +from unittest.signals import installHandler + +assert sys.version >= '3.3', 'Please use Python 3.3 or higher.' + +ARGS = argparse.ArgumentParser(description="Run all unittests.") +ARGS.add_argument( + '-v', action="store", dest='verbose', + nargs='?', const=1, type=int, default=0, help='verbose') +ARGS.add_argument( + '-x', action="store_true", dest='exclude', help='exclude tests') +ARGS.add_argument( + '-f', '--failfast', action="store_true", default=False, + dest='failfast', help='Stop on first fail or error') +ARGS.add_argument( + '-c', '--catch', action="store_true", default=False, + dest='catchbreak', help='Catch control-C and display results') +ARGS.add_argument( + '--forever', action="store_true", dest='forever', default=False, + help='run tests forever to catch sporadic errors') +ARGS.add_argument( + '--findleaks', action='store_true', dest='findleaks', + help='detect tests that leak memory') +ARGS.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.') +ARGS.add_argument('--seed', type=int, + help='random seed to reproduce a previous random run') +ARGS.add_argument( + '-q', action="store_true", dest='quiet', help='quiet') +ARGS.add_argument( + '--tests', action="store", dest='testsdir', default='tests', + help='tests directory') +ARGS.add_argument( + '--coverage', action="store_true", dest='coverage', + help='enable html coverage report') +ARGS.add_argument( + 'pattern', action="store", nargs="*", + help='optional regex patterns to match test ids (default all tests)') + +COV_ARGS = argparse.ArgumentParser(description="Run all unittests.") +COV_ARGS.add_argument( + '--coverage', action="store", dest='coverage', nargs='?', const='', + help='enable coverage report and provide python files directory') + + +def load_modules(basedir, suffix='.py'): + def list_dir(prefix, dir): + files = [] + + modpath = os.path.join(dir, '__init__.py') + if os.path.isfile(modpath): + mod = os.path.split(dir)[-1] + files.append(('{}{}'.format(prefix, mod), modpath)) + + prefix = '{}{}.'.format(prefix, mod) + + for name in os.listdir(dir): + path = os.path.join(dir, name) + + if os.path.isdir(path): + files.extend(list_dir('{}{}.'.format(prefix, name), path)) + else: + if (name != '__init__.py' and + name.endswith(suffix) and + not name.startswith(('.', '_'))): + files.append(('{}{}'.format(prefix, name[:-3]), path)) + + return files + + mods = [] + for modname, sourcefile in list_dir('', basedir): + if modname == 'runtests': + continue + if modname == 'test_pep492' and (sys.version_info < (3, 5)): + print("Skipping '{0}': need at least Python 3.5".format(modname), + file=sys.stderr) + continue + try: + loader = importlib.machinery.SourceFileLoader(modname, sourcefile) + mods.append((loader.load_module(), sourcefile)) + except SyntaxError: + raise + except unittest.SkipTest as err: + print("Skipping '{}': {}".format(modname, err), file=sys.stderr) + + return mods + + +def randomize_tests(tests, seed): + if seed is None: + seed = random.randrange(10000000) + random.seed(seed) + print("Randomize test execution order (seed: %s)" % seed) + random.shuffle(tests._tests) + + +class TestsFinder: + + def __init__(self, testsdir, includes=(), excludes=()): + self._testsdir = testsdir + self._includes = includes + self._excludes = excludes + self.find_available_tests() + + def find_available_tests(self): + """ + Find available test classes without instantiating them. + """ + self._test_factories = [] + mods = [mod for mod, _ in load_modules(self._testsdir)] + for mod in mods: + for name in set(dir(mod)): + if name.endswith('Tests'): + self._test_factories.append(getattr(mod, name)) + + def load_tests(self): + """ + Load test cases from the available test classes and apply + optional include / exclude filters. + """ + loader = unittest.TestLoader() + suite = unittest.TestSuite() + for test_factory in self._test_factories: + tests = loader.loadTestsFromTestCase(test_factory) + if self._includes: + tests = [test + for test in tests + if any(re.search(pat, test.id()) + for pat in self._includes)] + if self._excludes: + tests = [test + for test in tests + if not any(re.search(pat, test.id()) + for pat in self._excludes)] + suite.addTests(tests) + return suite + + +class TestResult(unittest.TextTestResult): + + def __init__(self, stream, descriptions, verbosity): + super().__init__(stream, descriptions, verbosity) + self.leaks = [] + + def startTest(self, test): + super().startTest(test) + gc.collect() + + def addSuccess(self, test): + super().addSuccess(test) + gc.collect() + if gc.garbage: + if self.showAll: + self.stream.writeln( + " Warning: test created {} uncollectable " + "object(s).".format(len(gc.garbage))) + # move the uncollectable objects somewhere so we don't see + # them again + self.leaks.append((self.getDescription(test), gc.garbage[:])) + del gc.garbage[:] + + +class TestRunner(unittest.TextTestRunner): + resultclass = TestResult + + def run(self, test): + result = super().run(test) + if result.leaks: + self.stream.writeln("{} tests leaks:".format(len(result.leaks))) + for name, leaks in result.leaks: + self.stream.writeln(' '*4 + name + ':') + for leak in leaks: + self.stream.writeln(' '*8 + repr(leak)) + return result + + +def _runtests(args, tests): + v = 0 if args.quiet else args.verbose + 1 + runner_factory = TestRunner if args.findleaks else unittest.TextTestRunner + if args.randomize: + randomize_tests(tests, args.seed) + runner = runner_factory(verbosity=v, failfast=args.failfast) + sys.stdout.flush() + sys.stderr.flush() + return runner.run(tests) + + +def runtests(): + # Print all warnings to the stdout. + warnings.simplefilter("always") + + args = ARGS.parse_args() + + if args.coverage and coverage is None: + URL = "bitbucket.org/pypa/setuptools/raw/bootstrap/ez_setup.py" + print(textwrap.dedent(""" + coverage package is not installed. + + To install coverage3 for Python 3, you need: + - Setuptools (https://pypi.python.org/pypi/setuptools) + + What worked for me: + - download {0} + * curl -O https://{0} + - python3 ez_setup.py + - python3 -m easy_install coverage + """.format(URL)).strip()) + sys.exit(1) + + testsdir = os.path.abspath(args.testsdir) + if not os.path.isdir(testsdir): + print("Tests directory is not found: {}\n".format(testsdir)) + ARGS.print_help() + return + + excludes = includes = [] + if args.exclude: + excludes = args.pattern + else: + includes = args.pattern + + v = 0 if args.quiet else args.verbose + 1 + failfast = args.failfast + + if args.coverage: + cov = coverage.coverage(branch=True, + source=['asyncio'], + ) + cov.start() + + logger = logging.getLogger() + if v == 0: + level = logging.CRITICAL + elif v == 1: + level = logging.ERROR + elif v == 2: + level = logging.WARNING + elif v == 3: + level = logging.INFO + elif v >= 4: + level = logging.DEBUG + logging.basicConfig(level=level) + + finder = TestsFinder(args.testsdir, includes, excludes) + if args.catchbreak: + installHandler() + import asyncio.coroutines + if asyncio.coroutines._DEBUG: + print("Run tests in debug mode") + else: + print("Run tests in release mode") + try: + tests = finder.load_tests() + if args.forever: + while True: + result = _runtests(args, tests) + if not result.wasSuccessful(): + sys.exit(1) + else: + result = _runtests(args, tests) + sys.exit(not result.wasSuccessful()) + finally: + if args.coverage: + cov.stop() + cov.save() + cov.html_report(directory='htmlcov') + print("\nCoverage report:") + cov.report(show_missing=False) + here = os.path.dirname(os.path.abspath(__file__)) + print("\nFor html report:") + print("open file://{}/htmlcov/index.html".format(here)) + + +if __name__ == '__main__': + runtests() diff --git a/thirdparty/asyncio/setup.py b/thirdparty/asyncio/setup.py new file mode 100644 index 0000000..5574cd0 --- /dev/null +++ b/thirdparty/asyncio/setup.py @@ -0,0 +1,53 @@ +# Release procedure: +# - run tox (to run runtests.py and run_aiotest.py) +# - maybe test examples +# - update version in setup.py +# - hg ci +# - hg tag VERSION +# - hg push +# - run on Linux: python setup.py register sdist upload +# - run on Windows: python release.py VERSION +# - increment version in setup.py +# - hg ci && hg push + +import os +import sys +try: + from setuptools import setup, Extension +except ImportError: + # Use distutils.core as a fallback. + # We won't be able to build the Wheel file on Windows. + from distutils.core import setup, Extension + +if sys.version_info < (3, 3, 0): + raise RuntimeError("asyncio requires Python 3.3.0+") + +extensions = [] +if os.name == 'nt': + ext = Extension( + 'asyncio._overlapped', ['overlapped.c'], libraries=['ws2_32'], + ) + extensions.append(ext) + +with open("README.rst") as fp: + long_description = fp.read() + +setup( + name="asyncio", + version="3.4.4", + + description="reference implementation of PEP 3156", + long_description=long_description, + url="http://www.python.org/dev/peps/pep-3156/", + + classifiers=[ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + ], + + packages=["asyncio"], + test_suite="runtests.runtests", + + ext_modules=extensions, +) diff --git a/thirdparty/asyncio/tests/echo.py b/thirdparty/asyncio/tests/echo.py new file mode 100644 index 0000000..006364b --- /dev/null +++ b/thirdparty/asyncio/tests/echo.py @@ -0,0 +1,8 @@ +import os + +if __name__ == '__main__': + while True: + buf = os.read(0, 1024) + if not buf: + break + os.write(1, buf) diff --git a/thirdparty/asyncio/tests/echo2.py b/thirdparty/asyncio/tests/echo2.py new file mode 100644 index 0000000..e83ca09 --- /dev/null +++ b/thirdparty/asyncio/tests/echo2.py @@ -0,0 +1,6 @@ +import os + +if __name__ == '__main__': + buf = os.read(0, 1024) + os.write(1, b'OUT:'+buf) + os.write(2, b'ERR:'+buf) diff --git a/thirdparty/asyncio/tests/echo3.py b/thirdparty/asyncio/tests/echo3.py new file mode 100644 index 0000000..0644967 --- /dev/null +++ b/thirdparty/asyncio/tests/echo3.py @@ -0,0 +1,11 @@ +import os + +if __name__ == '__main__': + while True: + buf = os.read(0, 1024) + if not buf: + break + try: + os.write(1, b'OUT:'+buf) + except OSError as ex: + os.write(2, b'ERR:' + ex.__class__.__name__.encode('ascii')) diff --git a/thirdparty/asyncio/tests/keycert3.pem b/thirdparty/asyncio/tests/keycert3.pem new file mode 100644 index 0000000..5bfa62c --- /dev/null +++ b/thirdparty/asyncio/tests/keycert3.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP +jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM +9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ +aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe +yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j +y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ +AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW +5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL +9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 +1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT +DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh +1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m +JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 +RnJdHOMXWem7/w== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: + 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: + c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: + 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: + f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: + 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: + f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: + af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: + 21:82:a5:3c:88:e5:be:1b:b1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: + e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: + f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: + e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: + d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: + 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: + ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: + 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: + 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: + 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: + 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: + f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: + 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: + a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: + fc:a9:94:71 +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C +tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola +N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 +TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR +iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG +xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo +5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv +mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF +YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh +2EJ36/yplHE= +-----END CERTIFICATE----- diff --git a/thirdparty/asyncio/tests/pycacert.pem b/thirdparty/asyncio/tests/pycacert.pem new file mode 100644 index 0000000..09b1f3e --- /dev/null +++ b/thirdparty/asyncio/tests/pycacert.pem @@ -0,0 +1,78 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Jan 2 19:47:07 2023 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: + 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: + e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: + e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: + 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: + 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: + a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: + e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: + 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: + 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: + e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: + c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: + cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: + 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: + 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: + 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: + e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: + c5:4d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + X509v3 Authority Key Identifier: + keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: + 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: + a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: + 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: + 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: + 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: + fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: + 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: + 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: + 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: + ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: + 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: + b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: + 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: + 5e:58:c8:9e +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/thirdparty/asyncio/tests/sample.crt b/thirdparty/asyncio/tests/sample.crt new file mode 100644 index 0000000..6a1e3f3 --- /dev/null +++ b/thirdparty/asyncio/tests/sample.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICMzCCAZwCCQDFl4ys0fU7iTANBgkqhkiG9w0BAQUFADBeMQswCQYDVQQGEwJV +UzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuLUZyYW5jaXNjbzEi +MCAGA1UECgwZUHl0aG9uIFNvZnR3YXJlIEZvbmRhdGlvbjAeFw0xMzAzMTgyMDA3 +MjhaFw0yMzAzMTYyMDA3MjhaMF4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxp +Zm9ybmlhMRYwFAYDVQQHDA1TYW4tRnJhbmNpc2NvMSIwIAYDVQQKDBlQeXRob24g +U29mdHdhcmUgRm9uZGF0aW9uMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCn +t3s+J7L0xP/YdAQOacpPi9phlrzKZhcXL3XMu2LCUg2fNJpx/47Vc5TZSaO11uO7 +gdwVz3Z7Q2epAgwo59JLffLt5fia8+a/SlPweI/j4+wcIIIiqusnLfpqR8cIAavg +Z06cLYCDvb9wMlheIvSJY12skc1nnphWS2YJ0Xm6uQIDAQABMA0GCSqGSIb3DQEB +BQUAA4GBAE9PknG6pv72+5z/gsDGYy8sK5UNkbWSNr4i4e5lxVsF03+/M71H+3AB +MxVX4+A+Vlk2fmU+BrdHIIUE0r1dDcO3josQ9hc9OJpp5VLSQFP8VeuJCmzYPp9I +I8WbW93cnXnChTrYQVdgVoFdv7GE9YgU7NYkrGIM0nZl1/f/bHPB +-----END CERTIFICATE----- diff --git a/thirdparty/asyncio/tests/sample.key b/thirdparty/asyncio/tests/sample.key new file mode 100644 index 0000000..edfea8d --- /dev/null +++ b/thirdparty/asyncio/tests/sample.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQCnt3s+J7L0xP/YdAQOacpPi9phlrzKZhcXL3XMu2LCUg2fNJpx +/47Vc5TZSaO11uO7gdwVz3Z7Q2epAgwo59JLffLt5fia8+a/SlPweI/j4+wcIIIi +qusnLfpqR8cIAavgZ06cLYCDvb9wMlheIvSJY12skc1nnphWS2YJ0Xm6uQIDAQAB +AoGABfm8k19Yue3W68BecKEGS0VBV57GRTPT+MiBGvVGNIQ15gk6w3sGfMZsdD1y +bsUkQgcDb2d/4i5poBTpl/+Cd41V+c20IC/sSl5X1IEreHMKSLhy/uyjyiyfXlP1 +iXhToFCgLWwENWc8LzfUV8vuAV5WG6oL9bnudWzZxeqx8V0CQQDR7xwVj6LN70Eb +DUhSKLkusmFw5Gk9NJ/7wZ4eHg4B8c9KNVvSlLCLhcsVTQXuqYeFpOqytI45SneP +lr0vrvsDAkEAzITYiXu6ox5huDCG7imX2W9CAYuX638urLxBqBXMS7GqBzojD6RL +21Q8oPwJWJquERa3HDScq1deiQbM9uKIkwJBAIa1PLslGN216Xv3UPHPScyKD/aF +ynXIv+OnANPoiyp6RH4ksQ/18zcEGiVH8EeNpvV9tlAHhb+DZibQHgNr74sCQQC0 +zhToplu/bVKSlUQUNO0rqrI9z30FErDewKeCw5KSsIRSU1E/uM3fHr9iyq4wiL6u +GNjUtKZ0y46lsT9uW6LFAkB5eqeEQnshAdr3X5GykWHJ8DDGBXPPn6Rce1NX4RSq +V9khG2z1bFyfo+hMqpYnF2k32hVq3E54RS8YYnwBsVof +-----END RSA PRIVATE KEY----- diff --git a/thirdparty/asyncio/tests/ssl_cert.pem b/thirdparty/asyncio/tests/ssl_cert.pem new file mode 100644 index 0000000..47a7d7e --- /dev/null +++ b/thirdparty/asyncio/tests/ssl_cert.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/thirdparty/asyncio/tests/ssl_key.pem b/thirdparty/asyncio/tests/ssl_key.pem new file mode 100644 index 0000000..3fd3bbd --- /dev/null +++ b/thirdparty/asyncio/tests/ssl_key.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- diff --git a/thirdparty/asyncio/tests/test_base_events.py b/thirdparty/asyncio/tests/test_base_events.py new file mode 100644 index 0000000..cdbd587 --- /dev/null +++ b/thirdparty/asyncio/tests/test_base_events.py @@ -0,0 +1,1716 @@ +"""Tests for base_events.py""" + +import errno +import logging +import math +import os +import socket +import sys +import threading +import time +import unittest +from unittest import mock + +import asyncio +from asyncio import base_events +from asyncio import constants +from asyncio import test_utils +try: + from test import support +except ImportError: + from asyncio import test_support as support +try: + from test.support.script_helper import assert_python_ok +except ImportError: + try: + from test.script_helper import assert_python_ok + except ImportError: + from asyncio.test_support import assert_python_ok + + +MOCK_ANY = mock.ANY +PY34 = sys.version_info >= (3, 4) + + +def mock_socket_module(): + m_socket = mock.MagicMock(spec=socket) + for name in ( + 'AF_INET', 'AF_INET6', 'AF_UNSPEC', 'IPPROTO_TCP', 'IPPROTO_UDP', + 'SOCK_STREAM', 'SOCK_DGRAM', 'SOL_SOCKET', 'SO_REUSEADDR', 'inet_pton' + ): + if hasattr(socket, name): + setattr(m_socket, name, getattr(socket, name)) + else: + delattr(m_socket, name) + + m_socket.socket = mock.MagicMock() + m_socket.socket.return_value = test_utils.mock_nonblocking_socket() + m_socket.getaddrinfo._is_coroutine = False + + return m_socket + + +def patch_socket(f): + return mock.patch('asyncio.base_events.socket', + new_callable=mock_socket_module)(f) + + +class BaseEventTests(test_utils.TestCase): + + def test_ipaddr_info(self): + UNSPEC = socket.AF_UNSPEC + INET = socket.AF_INET + INET6 = socket.AF_INET6 + STREAM = socket.SOCK_STREAM + DGRAM = socket.SOCK_DGRAM + TCP = socket.IPPROTO_TCP + UDP = socket.IPPROTO_UDP + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info(b'1.2.3.4', 1, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, STREAM, TCP)) + + self.assertEqual( + (INET, DGRAM, UDP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, DGRAM, UDP)) + + # Socket type STREAM implies TCP protocol. + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, STREAM, 0)) + + # Socket type DGRAM implies UDP protocol. + self.assertEqual( + (INET, DGRAM, UDP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, DGRAM, 0)) + + # No socket type. + self.assertIsNone( + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, 0, 0)) + + # IPv4 address with family IPv6. + self.assertIsNone( + base_events._ipaddr_info('1.2.3.4', 1, INET6, STREAM, TCP)) + + self.assertEqual( + (INET6, STREAM, TCP, '', ('::3', 1)), + base_events._ipaddr_info('::3', 1, INET6, STREAM, TCP)) + + self.assertEqual( + (INET6, STREAM, TCP, '', ('::3', 1)), + base_events._ipaddr_info('::3', 1, UNSPEC, STREAM, TCP)) + + # IPv6 address with family IPv4. + self.assertIsNone( + base_events._ipaddr_info('::3', 1, INET, STREAM, TCP)) + + # IPv6 address with zone index. + self.assertIsNone( + base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP)) + + def test_port_parameter_types(self): + # Test obscure kinds of arguments for "port". + INET = socket.AF_INET + STREAM = socket.SOCK_STREAM + TCP = socket.IPPROTO_TCP + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', None, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', b'', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', '', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', '1', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', b'1', INET, STREAM, TCP)) + + @patch_socket + def test_ipaddr_info_no_inet_pton(self, m_socket): + del m_socket.inet_pton + self.assertIsNone(base_events._ipaddr_info('1.2.3.4', 1, + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP)) + + +class BaseEventLoopTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = base_events.BaseEventLoop() + self.loop._selector = mock.Mock() + self.loop._selector.select.return_value = () + self.set_event_loop(self.loop) + + def test_not_implemented(self): + m = mock.Mock() + self.assertRaises( + NotImplementedError, + self.loop._make_socket_transport, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_ssl_transport, m, m, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_datagram_transport, m, m) + self.assertRaises( + NotImplementedError, self.loop._process_events, []) + self.assertRaises( + NotImplementedError, self.loop._write_to_self) + self.assertRaises( + NotImplementedError, + self.loop._make_read_pipe_transport, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_write_pipe_transport, m, m) + gen = self.loop._make_subprocess_transport(m, m, m, m, m, m, m) + with self.assertRaises(NotImplementedError): + gen.send(None) + + def test_close(self): + self.assertFalse(self.loop.is_closed()) + self.loop.close() + self.assertTrue(self.loop.is_closed()) + + # it should be possible to call close() more than once + self.loop.close() + self.loop.close() + + # operation blocked when the loop is closed + f = asyncio.Future(loop=self.loop) + self.assertRaises(RuntimeError, self.loop.run_forever) + self.assertRaises(RuntimeError, self.loop.run_until_complete, f) + + def test__add_callback_handle(self): + h = asyncio.Handle(lambda: False, (), self.loop) + + self.loop._add_callback(h) + self.assertFalse(self.loop._scheduled) + self.assertIn(h, self.loop._ready) + + def test__add_callback_cancelled_handle(self): + h = asyncio.Handle(lambda: False, (), self.loop) + h.cancel() + + self.loop._add_callback(h) + self.assertFalse(self.loop._scheduled) + self.assertFalse(self.loop._ready) + + def test_set_default_executor(self): + executor = mock.Mock() + self.loop.set_default_executor(executor) + self.assertIs(executor, self.loop._default_executor) + + def test_getnameinfo(self): + sockaddr = mock.Mock() + self.loop.run_in_executor = mock.Mock() + self.loop.getnameinfo(sockaddr) + self.assertEqual( + (None, socket.getnameinfo, sockaddr, 0), + self.loop.run_in_executor.call_args[0]) + + def test_call_soon(self): + def cb(): + pass + + h = self.loop.call_soon(cb) + self.assertEqual(h._callback, cb) + self.assertIsInstance(h, asyncio.Handle) + self.assertIn(h, self.loop._ready) + + def test_call_soon_non_callable(self): + self.loop.set_debug(True) + with self.assertRaisesRegex(TypeError, 'a callable object'): + self.loop.call_soon(1) + + def test_call_later(self): + def cb(): + pass + + h = self.loop.call_later(10.0, cb) + self.assertIsInstance(h, asyncio.TimerHandle) + self.assertIn(h, self.loop._scheduled) + self.assertNotIn(h, self.loop._ready) + + def test_call_later_negative_delays(self): + calls = [] + + def cb(arg): + calls.append(arg) + + self.loop._process_events = mock.Mock() + self.loop.call_later(-1, cb, 'a') + self.loop.call_later(-2, cb, 'b') + test_utils.run_briefly(self.loop) + self.assertEqual(calls, ['b', 'a']) + + def test_time_and_call_at(self): + def cb(): + self.loop.stop() + + self.loop._process_events = mock.Mock() + delay = 0.1 + + when = self.loop.time() + delay + self.loop.call_at(when, cb) + t0 = self.loop.time() + self.loop.run_forever() + dt = self.loop.time() - t0 + + # 50 ms: maximum granularity of the event loop + self.assertGreaterEqual(dt, delay - 0.050, dt) + # tolerate a difference of +800 ms because some Python buildbots + # are really slow + self.assertLessEqual(dt, 0.9, dt) + + def check_thread(self, loop, debug): + def cb(): + pass + + loop.set_debug(debug) + if debug: + msg = ("Non-thread-safe operation invoked on an event loop other " + "than the current one") + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_soon(cb) + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_later(60, cb) + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_at(loop.time() + 60, cb) + else: + loop.call_soon(cb) + loop.call_later(60, cb) + loop.call_at(loop.time() + 60, cb) + + def test_check_thread(self): + def check_in_thread(loop, event, debug, create_loop, fut): + # wait until the event loop is running + event.wait() + + try: + if create_loop: + loop2 = base_events.BaseEventLoop() + try: + asyncio.set_event_loop(loop2) + self.check_thread(loop, debug) + finally: + asyncio.set_event_loop(None) + loop2.close() + else: + self.check_thread(loop, debug) + except Exception as exc: + loop.call_soon_threadsafe(fut.set_exception, exc) + else: + loop.call_soon_threadsafe(fut.set_result, None) + + def test_thread(loop, debug, create_loop=False): + event = threading.Event() + fut = asyncio.Future(loop=loop) + loop.call_soon(event.set) + args = (loop, event, debug, create_loop, fut) + thread = threading.Thread(target=check_in_thread, args=args) + thread.start() + loop.run_until_complete(fut) + thread.join() + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + + # raise RuntimeError if the thread has no event loop + test_thread(self.loop, True) + + # check disabled if debug mode is disabled + test_thread(self.loop, False) + + # raise RuntimeError if the event loop of the thread is not the called + # event loop + test_thread(self.loop, True, create_loop=True) + + # check disabled if debug mode is disabled + test_thread(self.loop, False, create_loop=True) + + def test_run_once_in_executor_plain(self): + def cb(): + pass + f = asyncio.Future(loop=self.loop) + executor = mock.Mock() + executor.submit.return_value = f + + self.loop.set_default_executor(executor) + + res = self.loop.run_in_executor(None, cb) + self.assertIs(f, res) + + executor = mock.Mock() + executor.submit.return_value = f + res = self.loop.run_in_executor(executor, cb) + self.assertIs(f, res) + self.assertTrue(executor.submit.called) + + f.cancel() # Don't complain about abandoned Future. + + def test__run_once(self): + h1 = asyncio.TimerHandle(time.monotonic() + 5.0, lambda: True, (), + self.loop) + h2 = asyncio.TimerHandle(time.monotonic() + 10.0, lambda: True, (), + self.loop) + + h1.cancel() + + self.loop._process_events = mock.Mock() + self.loop._scheduled.append(h1) + self.loop._scheduled.append(h2) + self.loop._run_once() + + t = self.loop._selector.select.call_args[0][0] + self.assertTrue(9.5 < t < 10.5, t) + self.assertEqual([h2], self.loop._scheduled) + self.assertTrue(self.loop._process_events.called) + + def test_set_debug(self): + self.loop.set_debug(True) + self.assertTrue(self.loop.get_debug()) + self.loop.set_debug(False) + self.assertFalse(self.loop.get_debug()) + + @mock.patch('asyncio.base_events.logger') + def test__run_once_logging(self, m_logger): + def slow_select(timeout): + # Sleep a bit longer than a second to avoid timer resolution + # issues. + time.sleep(1.1) + return [] + + # logging needs debug flag + self.loop.set_debug(True) + + # Log to INFO level if timeout > 1.0 sec. + self.loop._selector.select = slow_select + self.loop._process_events = mock.Mock() + self.loop._run_once() + self.assertEqual(logging.INFO, m_logger.log.call_args[0][0]) + + def fast_select(timeout): + time.sleep(0.001) + return [] + + self.loop._selector.select = fast_select + self.loop._run_once() + self.assertEqual(logging.DEBUG, m_logger.log.call_args[0][0]) + + def test__run_once_schedule_handle(self): + handle = None + processed = False + + def cb(loop): + nonlocal processed, handle + processed = True + handle = loop.call_soon(lambda: True) + + h = asyncio.TimerHandle(time.monotonic() - 1, cb, (self.loop,), + self.loop) + + self.loop._process_events = mock.Mock() + self.loop._scheduled.append(h) + self.loop._run_once() + + self.assertTrue(processed) + self.assertEqual([handle], list(self.loop._ready)) + + def test__run_once_cancelled_event_cleanup(self): + self.loop._process_events = mock.Mock() + + self.assertTrue( + 0 < base_events._MIN_CANCELLED_TIMER_HANDLES_FRACTION < 1.0) + + def cb(): + pass + + # Set up one "blocking" event that will not be cancelled to + # ensure later cancelled events do not make it to the head + # of the queue and get cleaned. + not_cancelled_count = 1 + self.loop.call_later(3000, cb) + + # Add less than threshold (base_events._MIN_SCHEDULED_TIMER_HANDLES) + # cancelled handles, ensure they aren't removed + + cancelled_count = 2 + for x in range(2): + h = self.loop.call_later(3600, cb) + h.cancel() + + # Add some cancelled events that will be at head and removed + cancelled_count += 2 + for x in range(2): + h = self.loop.call_later(100, cb) + h.cancel() + + # This test is invalid if _MIN_SCHEDULED_TIMER_HANDLES is too low + self.assertLessEqual(cancelled_count + not_cancelled_count, + base_events._MIN_SCHEDULED_TIMER_HANDLES) + + self.assertEqual(self.loop._timer_cancelled_count, cancelled_count) + + self.loop._run_once() + + cancelled_count -= 2 + + self.assertEqual(self.loop._timer_cancelled_count, cancelled_count) + + self.assertEqual(len(self.loop._scheduled), + cancelled_count + not_cancelled_count) + + # Need enough events to pass _MIN_CANCELLED_TIMER_HANDLES_FRACTION + # so that deletion of cancelled events will occur on next _run_once + add_cancel_count = int(math.ceil( + base_events._MIN_SCHEDULED_TIMER_HANDLES * + base_events._MIN_CANCELLED_TIMER_HANDLES_FRACTION)) + 1 + + add_not_cancel_count = max(base_events._MIN_SCHEDULED_TIMER_HANDLES - + add_cancel_count, 0) + + # Add some events that will not be cancelled + not_cancelled_count += add_not_cancel_count + for x in range(add_not_cancel_count): + self.loop.call_later(3600, cb) + + # Add enough cancelled events + cancelled_count += add_cancel_count + for x in range(add_cancel_count): + h = self.loop.call_later(3600, cb) + h.cancel() + + # Ensure all handles are still scheduled + self.assertEqual(len(self.loop._scheduled), + cancelled_count + not_cancelled_count) + + self.loop._run_once() + + # Ensure cancelled events were removed + self.assertEqual(len(self.loop._scheduled), not_cancelled_count) + + # Ensure only uncancelled events remain scheduled + self.assertTrue(all([not x._cancelled for x in self.loop._scheduled])) + + def test_run_until_complete_type_error(self): + self.assertRaises(TypeError, + self.loop.run_until_complete, 'blah') + + def test_run_until_complete_loop(self): + task = asyncio.Future(loop=self.loop) + other_loop = self.new_test_loop() + self.addCleanup(other_loop.close) + self.assertRaises(ValueError, + other_loop.run_until_complete, task) + + def test_subprocess_exec_invalid_args(self): + args = [sys.executable, '-c', 'pass'] + + # missing program parameter (empty args) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol) + + # expected multiple arguments, not a list + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, args) + + # program arguments must be strings, not int + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, sys.executable, 123) + + # universal_newlines, shell, bufsize must not be set + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, universal_newlines=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, shell=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, bufsize=4096) + + def test_subprocess_shell_invalid_args(self): + # expected a string, not an int or a list + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 123) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, [sys.executable, '-c', 'pass']) + + # universal_newlines, shell, bufsize must not be set + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', universal_newlines=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', shell=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', bufsize=4096) + + def test_default_exc_handler_callback(self): + self.loop._process_events = mock.Mock() + + def zero_error(fut): + fut.set_result(True) + 1/0 + + # Test call_soon (events.Handle) + with mock.patch('asyncio.base_events.logger') as log: + fut = asyncio.Future(loop=self.loop) + self.loop.call_soon(zero_error, fut) + fut.add_done_callback(lambda fut: self.loop.stop()) + self.loop.run_forever() + log.error.assert_called_with( + test_utils.MockPattern('Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + # Test call_later (events.TimerHandle) + with mock.patch('asyncio.base_events.logger') as log: + fut = asyncio.Future(loop=self.loop) + self.loop.call_later(0.01, zero_error, fut) + fut.add_done_callback(lambda fut: self.loop.stop()) + self.loop.run_forever() + log.error.assert_called_with( + test_utils.MockPattern('Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + def test_default_exc_handler_coro(self): + self.loop._process_events = mock.Mock() + + @asyncio.coroutine + def zero_error_coro(): + yield from asyncio.sleep(0.01, loop=self.loop) + 1/0 + + # Test Future.__del__ + with mock.patch('asyncio.base_events.logger') as log: + fut = asyncio.ensure_future(zero_error_coro(), loop=self.loop) + fut.add_done_callback(lambda *args: self.loop.stop()) + self.loop.run_forever() + fut = None # Trigger Future.__del__ or futures._TracebackLogger + support.gc_collect() + if PY34: + # Future.__del__ in Python 3.4 logs error with + # an actual exception context + log.error.assert_called_with( + test_utils.MockPattern('.*exception was never retrieved'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + else: + # futures._TracebackLogger logs only textual traceback + log.error.assert_called_with( + test_utils.MockPattern( + '.*exception was never retrieved.*ZeroDiv'), + exc_info=False) + + def test_set_exc_handler_invalid(self): + with self.assertRaisesRegex(TypeError, 'A callable object or None'): + self.loop.set_exception_handler('spam') + + def test_set_exc_handler_custom(self): + def zero_error(): + 1/0 + + def run_loop(): + handle = self.loop.call_soon(zero_error) + self.loop._run_once() + return handle + + self.loop.set_debug(True) + self.loop._process_events = mock.Mock() + + self.assertIsNone(self.loop.get_exception_handler()) + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + self.assertIs(self.loop.get_exception_handler(), mock_handler) + handle = run_loop() + mock_handler.assert_called_with(self.loop, { + 'exception': MOCK_ANY, + 'message': test_utils.MockPattern( + 'Exception in callback.*zero_error'), + 'handle': handle, + 'source_traceback': handle._source_traceback, + }) + mock_handler.reset_mock() + + self.loop.set_exception_handler(None) + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern( + 'Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + assert not mock_handler.called + + def test_set_exc_handler_broken(self): + def run_loop(): + def zero_error(): + 1/0 + self.loop.call_soon(zero_error) + self.loop._run_once() + + def handler(loop, context): + raise AttributeError('spam') + + self.loop._process_events = mock.Mock() + + self.loop.set_exception_handler(handler) + + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern( + 'Unhandled error in exception handler'), + exc_info=(AttributeError, MOCK_ANY, MOCK_ANY)) + + def test_default_exc_handler_broken(self): + _context = None + + class Loop(base_events.BaseEventLoop): + + _selector = mock.Mock() + _process_events = mock.Mock() + + def default_exception_handler(self, context): + nonlocal _context + _context = context + # Simulates custom buggy "default_exception_handler" + raise ValueError('spam') + + loop = Loop() + self.addCleanup(loop.close) + asyncio.set_event_loop(loop) + + def run_loop(): + def zero_error(): + 1/0 + loop.call_soon(zero_error) + loop._run_once() + + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + 'Exception in default exception handler', + exc_info=True) + + def custom_handler(loop, context): + raise ValueError('ham') + + _context = None + loop.set_exception_handler(custom_handler) + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern('Exception in default exception.*' + 'while handling.*in custom'), + exc_info=True) + + # Check that original context was passed to default + # exception handler. + self.assertIn('context', _context) + self.assertIs(type(_context['context']['exception']), + ZeroDivisionError) + + def test_set_task_factory_invalid(self): + with self.assertRaisesRegex( + TypeError, 'task factory must be a callable or None'): + + self.loop.set_task_factory(1) + + self.assertIsNone(self.loop.get_task_factory()) + + def test_set_task_factory(self): + self.loop._process_events = mock.Mock() + + class MyTask(asyncio.Task): + pass + + @asyncio.coroutine + def coro(): + pass + + factory = lambda loop, coro: MyTask(coro, loop=loop) + + self.assertIsNone(self.loop.get_task_factory()) + self.loop.set_task_factory(factory) + self.assertIs(self.loop.get_task_factory(), factory) + + task = self.loop.create_task(coro()) + self.assertTrue(isinstance(task, MyTask)) + self.loop.run_until_complete(task) + + self.loop.set_task_factory(None) + self.assertIsNone(self.loop.get_task_factory()) + + task = self.loop.create_task(coro()) + self.assertTrue(isinstance(task, asyncio.Task)) + self.assertFalse(isinstance(task, MyTask)) + self.loop.run_until_complete(task) + + def test_env_var_debug(self): + code = '\n'.join(( + 'import asyncio', + 'loop = asyncio.get_event_loop()', + 'print(loop.get_debug())')) + + # Test with -E to not fail if the unit test was run with + # PYTHONASYNCIODEBUG set to a non-empty string + sts, stdout, stderr = assert_python_ok('-E', '-c', code) + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='') + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='1') + self.assertEqual(stdout.rstrip(), b'True') + + sts, stdout, stderr = assert_python_ok('-E', '-c', code, + PYTHONASYNCIODEBUG='1') + self.assertEqual(stdout.rstrip(), b'False') + + def test_create_task(self): + class MyTask(asyncio.Task): + pass + + @asyncio.coroutine + def test(): + pass + + class EventLoop(base_events.BaseEventLoop): + def create_task(self, coro): + return MyTask(coro, loop=loop) + + loop = EventLoop() + self.set_event_loop(loop) + + coro = test() + task = asyncio.ensure_future(coro, loop=loop) + self.assertIsInstance(task, MyTask) + + # make warnings quiet + task._log_destroy_pending = False + coro.close() + + def test_run_forever_keyboard_interrupt(self): + # Python issue #22601: ensure that the temporary task created by + # run_forever() consumes the KeyboardInterrupt and so don't log + # a warning + @asyncio.coroutine + def raise_keyboard_interrupt(): + raise KeyboardInterrupt + + self.loop._process_events = mock.Mock() + self.loop.call_exception_handler = mock.Mock() + + try: + self.loop.run_until_complete(raise_keyboard_interrupt()) + except KeyboardInterrupt: + pass + self.loop.close() + support.gc_collect() + + self.assertFalse(self.loop.call_exception_handler.called) + + def test_run_until_complete_baseexception(self): + # Python issue #22429: run_until_complete() must not schedule a pending + # call to stop() if the future raised a BaseException + @asyncio.coroutine + def raise_keyboard_interrupt(): + raise KeyboardInterrupt + + self.loop._process_events = mock.Mock() + + try: + self.loop.run_until_complete(raise_keyboard_interrupt()) + except KeyboardInterrupt: + pass + + def func(): + self.loop.stop() + func.called = True + func.called = False + try: + self.loop.call_soon(func) + self.loop.run_forever() + except KeyboardInterrupt: + pass + self.assertTrue(func.called) + + def test_single_selecter_event_callback_after_stopping(self): + # Python issue #25593: A stopped event loop may cause event callbacks + # to run more than once. + event_sentinel = object() + callcount = 0 + doer = None + + def proc_events(event_list): + nonlocal doer + if event_sentinel in event_list: + doer = self.loop.call_soon(do_event) + + def do_event(): + nonlocal callcount + callcount += 1 + self.loop.call_soon(clear_selector) + + def clear_selector(): + doer.cancel() + self.loop._selector.select.return_value = () + + self.loop._process_events = proc_events + self.loop._selector.select.return_value = (event_sentinel,) + + for i in range(1, 3): + with self.subTest('Loop %d/2' % i): + self.loop.call_soon(self.loop.stop) + self.loop.run_forever() + self.assertEqual(callcount, 1) + + def test_run_once(self): + # Simple test for test_utils.run_once(). It may seem strange + # to have a test for this (the function isn't even used!) but + # it's a de-factor standard API for library tests. This tests + # the idiom: loop.call_soon(loop.stop); loop.run_forever(). + count = 0 + + def callback(): + nonlocal count + count += 1 + + self.loop._process_events = mock.Mock() + self.loop.call_soon(callback) + test_utils.run_once(self.loop) + self.assertEqual(count, 1) + + def test_run_forever_pre_stopped(self): + # Test that the old idiom for pre-stopping the loop works. + self.loop._process_events = mock.Mock() + self.loop.stop() + self.loop.run_forever() + self.loop._selector.select.assert_called_once_with(0) + + +class MyProto(asyncio.Protocol): + done = None + + def __init__(self, create_future=False): + self.state = 'INITIAL' + self.nbytes = 0 + if create_future: + self.done = asyncio.Future() + + def connection_made(self, transport): + self.transport = transport + assert self.state == 'INITIAL', self.state + self.state = 'CONNECTED' + transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n') + + def data_received(self, data): + assert self.state == 'CONNECTED', self.state + self.nbytes += len(data) + + def eof_received(self): + assert self.state == 'CONNECTED', self.state + self.state = 'EOF' + + def connection_lost(self, exc): + assert self.state in ('CONNECTED', 'EOF'), self.state + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyDatagramProto(asyncio.DatagramProtocol): + done = None + + def __init__(self, create_future=False, loop=None): + self.state = 'INITIAL' + self.nbytes = 0 + if create_future: + self.done = asyncio.Future(loop=loop) + + def connection_made(self, transport): + self.transport = transport + assert self.state == 'INITIAL', self.state + self.state = 'INITIALIZED' + + def datagram_received(self, data, addr): + assert self.state == 'INITIALIZED', self.state + self.nbytes += len(data) + + def error_received(self, exc): + assert self.state == 'INITIALIZED', self.state + + def connection_lost(self, exc): + assert self.state == 'INITIALIZED', self.state + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class BaseEventLoopWithSelectorTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + @patch_socket + def test_create_connection_multiple_errors(self, m_socket): + + class MyProto(asyncio.Protocol): + pass + + @asyncio.coroutine + def getaddrinfo(*args, **kw): + yield from [] + return [(2, 1, 6, '', ('107.6.106.82', 80)), + (2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop) + + idx = -1 + errors = ['err1', 'err2'] + + def _socket(*args, **kw): + nonlocal idx, errors + idx += 1 + raise OSError(errors[idx]) + + m_socket.socket = _socket + + self.loop.getaddrinfo = getaddrinfo_task + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(coro) + + self.assertEqual(str(cm.exception), 'Multiple exceptions: err1, err2') + + @patch_socket + def test_create_connection_timeout(self, m_socket): + # Ensure that the socket is closed on timeout + sock = mock.Mock() + m_socket.socket.return_value = sock + + def getaddrinfo(*args, **kw): + fut = asyncio.Future(loop=self.loop) + addr = (socket.AF_INET, socket.SOCK_STREAM, 0, '', + ('127.0.0.1', 80)) + fut.set_result([addr]) + return fut + self.loop.getaddrinfo = getaddrinfo + + with mock.patch.object(self.loop, 'sock_connect', + side_effect=asyncio.TimeoutError): + coro = self.loop.create_connection(MyProto, '127.0.0.1', 80) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + def test_create_connection_host_port_sock(self): + coro = self.loop.create_connection( + MyProto, 'example.com', 80, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_no_host_port_sock(self): + coro = self.loop.create_connection(MyProto) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_no_getaddrinfo(self): + @asyncio.coroutine + def getaddrinfo(*args, **kw): + yield from [] + + def getaddrinfo_task(*args, **kwds): + return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop) + + self.loop.getaddrinfo = getaddrinfo_task + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_connection_connect_err(self): + @asyncio.coroutine + def getaddrinfo(*args, **kw): + yield from [] + return [(2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_connection_multiple(self): + @asyncio.coroutine + def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('0.0.0.1', 80)), + (2, 1, 6, '', ('0.0.0.2', 80))] + + def getaddrinfo_task(*args, **kwds): + return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET) + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + + @patch_socket + def test_create_connection_multiple_errors_local_addr(self, m_socket): + + def bind(addr): + if addr[0] == '0.0.0.1': + err = OSError('Err') + err.strerror = 'Err' + raise err + + m_socket.socket.return_value.bind = bind + + @asyncio.coroutine + def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('0.0.0.1', 80)), + (2, 1, 6, '', ('0.0.0.2', 80))] + + def getaddrinfo_task(*args, **kwds): + return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError('Err2') + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, + local_addr=(None, 8080)) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(coro) + + self.assertTrue(str(cm.exception).startswith('Multiple exceptions: ')) + self.assertTrue(m_socket.socket.return_value.close.called) + + def _test_create_connection_ip_addr(self, m_socket, allow_inet_pton): + # Test the fallback code, even if this system has inet_pton. + if not allow_inet_pton: + del m_socket.inet_pton + + m_socket.getaddrinfo = socket.getaddrinfo + sock = m_socket.socket.return_value + + self.loop._add_reader = mock.Mock() + self.loop._add_reader._is_coroutine = False + self.loop._add_writer = mock.Mock() + self.loop._add_writer._is_coroutine = False + + coro = self.loop.create_connection(asyncio.Protocol, '1.2.3.4', 80) + t, p = self.loop.run_until_complete(coro) + try: + sock.connect.assert_called_with(('1.2.3.4', 80)) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + sock.family = socket.AF_INET6 + coro = self.loop.create_connection(asyncio.Protocol, '::1', 80) + t, p = self.loop.run_until_complete(coro) + try: + # Without inet_pton we use getaddrinfo, which transforms ('::1', 80) + # to ('::1', 80, 0, 0). The last 0s are flow info, scope id. + [address] = sock.connect.call_args[0] + host, port = address[:2] + self.assertRegex(host, r'::(0\.)*1') + self.assertEqual(port, 80) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET6) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + @patch_socket + def test_create_connection_ip_addr(self, m_socket): + self._test_create_connection_ip_addr(m_socket, True) + + @patch_socket + def test_create_connection_no_inet_pton(self, m_socket): + self._test_create_connection_ip_addr(m_socket, False) + + @patch_socket + def test_create_connection_service_name(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + sock = m_socket.socket.return_value + + self.loop._add_reader = mock.Mock() + self.loop._add_reader._is_coroutine = False + self.loop._add_writer = mock.Mock() + self.loop._add_writer._is_coroutine = False + + for service, port in ('http', 80), (b'http', 80): + coro = self.loop.create_connection(asyncio.Protocol, + '127.0.0.1', service) + + t, p = self.loop.run_until_complete(coro) + try: + sock.connect.assert_called_with(('127.0.0.1', port)) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + for service in 'nonsense', b'nonsense': + coro = self.loop.create_connection(asyncio.Protocol, + '127.0.0.1', service) + + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + + def test_create_connection_no_local_addr(self): + @asyncio.coroutine + def getaddrinfo(host, *args, **kw): + if host == 'example.com': + return [(2, 1, 6, '', ('107.6.106.82', 80)), + (2, 1, 6, '', ('107.6.106.82', 80))] + else: + return [] + + def getaddrinfo_task(*args, **kwds): + return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop) + self.loop.getaddrinfo = getaddrinfo_task + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, + local_addr=(None, 8080)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_connection_bluetooth(self, m_socket): + # See http://bugs.python.org/issue27136, fallback to getaddrinfo when + # we can't recognize an address is resolved, e.g. a Bluetooth address. + addr = ('00:01:02:03:04:05', 1) + + def getaddrinfo(host, port, *args, **kw): + assert (host, port) == addr + return [(999, 1, 999, '', (addr, 1))] + + m_socket.getaddrinfo = getaddrinfo + sock = m_socket.socket() + coro = self.loop.sock_connect(sock, addr) + self.loop.run_until_complete(coro) + + def test_create_connection_ssl_server_hostname_default(self): + self.loop.getaddrinfo = mock.Mock() + + def mock_getaddrinfo(*args, **kwds): + f = asyncio.Future(loop=self.loop) + f.set_result([(socket.AF_INET, socket.SOCK_STREAM, + socket.SOL_TCP, '', ('1.2.3.4', 80))]) + return f + + self.loop.getaddrinfo.side_effect = mock_getaddrinfo + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.return_value = () + self.loop._make_ssl_transport = mock.Mock() + + class _SelectorTransportMock: + _sock = None + + def get_extra_info(self, key): + return mock.Mock() + + def close(self): + self._sock.close() + + def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, + **kwds): + waiter.set_result(None) + transport = _SelectorTransportMock() + transport._sock = sock + return transport + + self.loop._make_ssl_transport.side_effect = mock_make_ssl_transport + ANY = mock.ANY + # First try the default server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True) + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='python.org') + # Next try an explicit server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True, + server_hostname='perl.com') + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='perl.com') + # Finally try an explicit empty server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True, + server_hostname='') + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with(ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='') + + def test_create_connection_no_ssl_server_hostname_errors(self): + # When not using ssl, server_hostname must be None. + coro = self.loop.create_connection(MyProto, 'python.org', 80, + server_hostname='') + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + coro = self.loop.create_connection(MyProto, 'python.org', 80, + server_hostname='python.org') + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_ssl_server_hostname_errors(self): + # When using ssl, server_hostname may be None if host is non-empty. + coro = self.loop.create_connection(MyProto, '', 80, ssl=True) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + coro = self.loop.create_connection(MyProto, None, 80, ssl=True) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + sock = socket.socket() + coro = self.loop.create_connection(MyProto, None, None, + ssl=True, sock=sock) + self.addCleanup(sock.close) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_server_empty_host(self): + # if host is empty string use None instead + host = object() + + @asyncio.coroutine + def getaddrinfo(*args, **kw): + nonlocal host + host = args[0] + yield from [] + + def getaddrinfo_task(*args, **kwds): + return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop) + + self.loop.getaddrinfo = getaddrinfo_task + fut = self.loop.create_server(MyProto, '', 0) + self.assertRaises(OSError, self.loop.run_until_complete, fut) + self.assertIsNone(host) + + def test_create_server_host_port_sock(self): + fut = self.loop.create_server( + MyProto, '0.0.0.0', 0, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + def test_create_server_no_host_port_sock(self): + fut = self.loop.create_server(MyProto) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + def test_create_server_no_getaddrinfo(self): + getaddrinfo = self.loop.getaddrinfo = mock.Mock() + getaddrinfo.return_value = [] + + f = self.loop.create_server(MyProto, 'python.org', 0) + self.assertRaises(OSError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_nosoreuseport(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + del m_socket.SO_REUSEPORT + m_socket.socket.return_value = mock.Mock() + + f = self.loop.create_server( + MyProto, '0.0.0.0', 0, reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_soreuseport_only_defined(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + m_socket.socket.return_value = mock.Mock() + m_socket.SO_REUSEPORT = -1 + + f = self.loop.create_server( + MyProto, '0.0.0.0', 0, reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_cant_bind(self, m_socket): + + class Err(OSError): + strerror = 'error' + + m_socket.getaddrinfo.return_value = [ + (2, 1, 6, '', ('127.0.0.1', 10100))] + m_socket.getaddrinfo._is_coroutine = False + m_sock = m_socket.socket.return_value = mock.Mock() + m_sock.bind.side_effect = Err + + fut = self.loop.create_server(MyProto, '0.0.0.0', 0) + self.assertRaises(OSError, self.loop.run_until_complete, fut) + self.assertTrue(m_sock.close.called) + + @patch_socket + def test_create_datagram_endpoint_no_addrinfo(self, m_socket): + m_socket.getaddrinfo.return_value = [] + m_socket.getaddrinfo._is_coroutine = False + + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('localhost', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_datagram_endpoint_addr_error(self): + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr='localhost') + self.assertRaises( + AssertionError, self.loop.run_until_complete, coro) + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('localhost', 1, 2, 3)) + self.assertRaises( + AssertionError, self.loop.run_until_complete, coro) + + def test_create_datagram_endpoint_connect_err(self): + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, remote_addr=('127.0.0.1', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_socket_err(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + m_socket.socket.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, family=socket.AF_INET) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, local_addr=('127.0.0.1', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_datagram_endpoint_no_matching_family(self): + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, + remote_addr=('127.0.0.1', 0), local_addr=('::1', 0)) + self.assertRaises( + ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_setblk_err(self, m_socket): + m_socket.socket.return_value.setblocking.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, family=socket.AF_INET) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + self.assertTrue( + m_socket.socket.return_value.close.called) + + def test_create_datagram_endpoint_noaddr_nofamily(self): + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_cant_bind(self, m_socket): + class Err(OSError): + pass + + m_socket.getaddrinfo = socket.getaddrinfo + m_sock = m_socket.socket.return_value = mock.Mock() + m_sock.bind.side_effect = Err + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, + local_addr=('127.0.0.1', 0), family=socket.AF_INET) + self.assertRaises(Err, self.loop.run_until_complete, fut) + self.assertTrue(m_sock.close.called) + + def test_create_datagram_endpoint_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(('127.0.0.1', 0)) + fut = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + sock=sock) + transport, protocol = self.loop.run_until_complete(fut) + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + def test_create_datagram_endpoint_sock_sockopts(self): + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('127.0.0.1', 0), sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, remote_addr=('127.0.0.1', 0), sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, family=1, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, proto=1, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, flags=1, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, reuse_address=True, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, reuse_port=True, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, allow_broadcast=True, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + def test_create_datagram_endpoint_sockopts(self): + # Socket options should not be applied unless asked for. + # SO_REUSEADDR defaults to on for UNIX. + # SO_REUSEPORT is not available on all platforms. + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + local_addr=('127.0.0.1', 0)) + transport, protocol = self.loop.run_until_complete(coro) + sock = transport.get_extra_info('socket') + + reuse_address_default_on = ( + os.name == 'posix' and sys.platform != 'cygwin') + reuseport_supported = hasattr(socket, 'SO_REUSEPORT') + + if reuse_address_default_on: + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR)) + else: + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR)) + if reuseport_supported: + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST)) + + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + local_addr=('127.0.0.1', 0), + reuse_address=True, + reuse_port=reuseport_supported, + allow_broadcast=True) + transport, protocol = self.loop.run_until_complete(coro) + sock = transport.get_extra_info('socket') + + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR)) + if reuseport_supported: + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST)) + + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @patch_socket + def test_create_datagram_endpoint_nosoreuseport(self, m_socket): + del m_socket.SO_REUSEPORT + m_socket.socket.return_value = mock.Mock() + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + local_addr=('127.0.0.1', 0), + reuse_address=False, + reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_ip_addr(self, m_socket): + def getaddrinfo(*args, **kw): + self.fail('should not have called getaddrinfo') + + m_socket.getaddrinfo = getaddrinfo + m_socket.socket.return_value.bind = bind = mock.Mock() + self.loop._add_reader = mock.Mock() + self.loop._add_reader._is_coroutine = False + + reuseport_supported = hasattr(socket, 'SO_REUSEPORT') + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + local_addr=('1.2.3.4', 0), + reuse_address=False, + reuse_port=reuseport_supported) + + t, p = self.loop.run_until_complete(coro) + try: + bind.assert_called_with(('1.2.3.4', 0)) + m_socket.socket.assert_called_with(family=m_socket.AF_INET, + proto=m_socket.IPPROTO_UDP, + type=m_socket.SOCK_DGRAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + def test_accept_connection_retry(self): + sock = mock.Mock() + sock.accept.side_effect = BlockingIOError() + + self.loop._accept_connection(MyProto, sock) + self.assertFalse(sock.close.called) + + @mock.patch('asyncio.base_events.logger') + def test_accept_connection_exception(self, m_log): + sock = mock.Mock() + sock.fileno.return_value = 10 + sock.accept.side_effect = OSError(errno.EMFILE, 'Too many open files') + self.loop._remove_reader = mock.Mock() + self.loop.call_later = mock.Mock() + + self.loop._accept_connection(MyProto, sock) + self.assertTrue(m_log.error.called) + self.assertFalse(sock.close.called) + self.loop._remove_reader.assert_called_with(10) + self.loop.call_later.assert_called_with(constants.ACCEPT_RETRY_DELAY, + # self.loop._start_serving + mock.ANY, + MyProto, sock, None, None, mock.ANY) + + def test_call_coroutine(self): + @asyncio.coroutine + def simple_coroutine(): + pass + + self.loop.set_debug(True) + coro_func = simple_coroutine + coro_obj = coro_func() + self.addCleanup(coro_obj.close) + for func in (coro_func, coro_obj): + with self.assertRaises(TypeError): + self.loop.call_soon(func) + with self.assertRaises(TypeError): + self.loop.call_soon_threadsafe(func) + with self.assertRaises(TypeError): + self.loop.call_later(60, func) + with self.assertRaises(TypeError): + self.loop.call_at(self.loop.time() + 60, func) + with self.assertRaises(TypeError): + self.loop.run_in_executor(None, func) + + @mock.patch('asyncio.base_events.logger') + def test_log_slow_callbacks(self, m_logger): + def stop_loop_cb(loop): + loop.stop() + + @asyncio.coroutine + def stop_loop_coro(loop): + yield from () + loop.stop() + + asyncio.set_event_loop(self.loop) + self.loop.set_debug(True) + self.loop.slow_callback_duration = 0.0 + + # slow callback + self.loop.call_soon(stop_loop_cb, self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing " + "took .* seconds$") + + # slow task + asyncio.ensure_future(stop_loop_coro(self.loop), loop=self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing " + "took .* seconds$") + + +class RunningLoopTests(unittest.TestCase): + + def test_running_loop_within_a_loop(self): + @asyncio.coroutine + def runner(loop): + loop.run_forever() + + loop = asyncio.new_event_loop() + outer_loop = asyncio.new_event_loop() + try: + with self.assertRaisesRegex(RuntimeError, + 'while another loop is running'): + outer_loop.run_until_complete(runner(loop)) + finally: + loop.close() + outer_loop.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_events.py b/thirdparty/asyncio/tests/test_events.py new file mode 100644 index 0000000..4c18300 --- /dev/null +++ b/thirdparty/asyncio/tests/test_events.py @@ -0,0 +1,2747 @@ +"""Tests for events.py.""" + +import collections.abc +import functools +import gc +import io +import os +import platform +import re +import signal +import socket +try: + import ssl +except ImportError: + ssl = None +import subprocess +import sys +import threading +import time +import errno +import unittest +from unittest import mock +import weakref + +if sys.platform != 'win32': + import tty + +import asyncio +from asyncio import coroutines +from asyncio import proactor_events +from asyncio import selector_events +from asyncio import sslproto +from asyncio import test_utils +try: + from test import support +except ImportError: + from asyncio import test_support as support + + +def data_file(filename): + if hasattr(support, 'TEST_HOME_DIR'): + fullname = os.path.join(support.TEST_HOME_DIR, filename) + if os.path.isfile(fullname): + return fullname + fullname = os.path.join(os.path.dirname(__file__), filename) + if os.path.isfile(fullname): + return fullname + raise FileNotFoundError(filename) + + +def osx_tiger(): + """Return True if the platform is Mac OS 10.4 or older.""" + if sys.platform != 'darwin': + return False + version = platform.mac_ver()[0] + version = tuple(map(int, version.split('.'))) + return version < (10, 5) + + +ONLYCERT = data_file('ssl_cert.pem') +ONLYKEY = data_file('ssl_key.pem') +SIGNED_CERTFILE = data_file('keycert3.pem') +SIGNING_CA = data_file('pycacert.pem') +PEERCERT = {'serialNumber': 'B09264B1F2DA21D1', + 'version': 1, + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Nov 13 19:47:07 2022 GMT', + 'notBefore': 'Jan 4 19:47:07 2013 GMT'} + + +class MyBaseProto(asyncio.Protocol): + connected = None + done = None + + def __init__(self, loop=None): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = asyncio.Future(loop=loop) + self.done = asyncio.Future(loop=loop) + + def connection_made(self, transport): + self.transport = transport + assert self.state == 'INITIAL', self.state + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + + def data_received(self, data): + assert self.state == 'CONNECTED', self.state + self.nbytes += len(data) + + def eof_received(self): + assert self.state == 'CONNECTED', self.state + self.state = 'EOF' + + def connection_lost(self, exc): + assert self.state in ('CONNECTED', 'EOF'), self.state + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyProto(MyBaseProto): + def connection_made(self, transport): + super().connection_made(transport) + transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n') + + +class MyDatagramProto(asyncio.DatagramProtocol): + done = None + + def __init__(self, loop=None): + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.done = asyncio.Future(loop=loop) + + def connection_made(self, transport): + self.transport = transport + assert self.state == 'INITIAL', self.state + self.state = 'INITIALIZED' + + def datagram_received(self, data, addr): + assert self.state == 'INITIALIZED', self.state + self.nbytes += len(data) + + def error_received(self, exc): + assert self.state == 'INITIALIZED', self.state + + def connection_lost(self, exc): + assert self.state == 'INITIALIZED', self.state + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyReadPipeProto(asyncio.Protocol): + done = None + + def __init__(self, loop=None): + self.state = ['INITIAL'] + self.nbytes = 0 + self.transport = None + if loop is not None: + self.done = asyncio.Future(loop=loop) + + def connection_made(self, transport): + self.transport = transport + assert self.state == ['INITIAL'], self.state + self.state.append('CONNECTED') + + def data_received(self, data): + assert self.state == ['INITIAL', 'CONNECTED'], self.state + self.nbytes += len(data) + + def eof_received(self): + assert self.state == ['INITIAL', 'CONNECTED'], self.state + self.state.append('EOF') + + def connection_lost(self, exc): + if 'EOF' not in self.state: + self.state.append('EOF') # It is okay if EOF is missed. + assert self.state == ['INITIAL', 'CONNECTED', 'EOF'], self.state + self.state.append('CLOSED') + if self.done: + self.done.set_result(None) + + +class MyWritePipeProto(asyncio.BaseProtocol): + done = None + + def __init__(self, loop=None): + self.state = 'INITIAL' + self.transport = None + if loop is not None: + self.done = asyncio.Future(loop=loop) + + def connection_made(self, transport): + self.transport = transport + assert self.state == 'INITIAL', self.state + self.state = 'CONNECTED' + + def connection_lost(self, exc): + assert self.state == 'CONNECTED', self.state + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MySubprocessProtocol(asyncio.SubprocessProtocol): + + def __init__(self, loop): + self.state = 'INITIAL' + self.transport = None + self.connected = asyncio.Future(loop=loop) + self.completed = asyncio.Future(loop=loop) + self.disconnects = {fd: asyncio.Future(loop=loop) for fd in range(3)} + self.data = {1: b'', 2: b''} + self.returncode = None + self.got_data = {1: asyncio.Event(loop=loop), + 2: asyncio.Event(loop=loop)} + + def connection_made(self, transport): + self.transport = transport + assert self.state == 'INITIAL', self.state + self.state = 'CONNECTED' + self.connected.set_result(None) + + def connection_lost(self, exc): + assert self.state == 'CONNECTED', self.state + self.state = 'CLOSED' + self.completed.set_result(None) + + def pipe_data_received(self, fd, data): + assert self.state == 'CONNECTED', self.state + self.data[fd] += data + self.got_data[fd].set() + + def pipe_connection_lost(self, fd, exc): + assert self.state == 'CONNECTED', self.state + if exc: + self.disconnects[fd].set_exception(exc) + else: + self.disconnects[fd].set_result(exc) + + def process_exited(self): + assert self.state == 'CONNECTED', self.state + self.returncode = self.transport.get_returncode() + + +class EventLoopTestsMixin: + + def setUp(self): + super().setUp() + self.loop = self.create_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.loop.close() + gc.collect() + super().tearDown() + + def test_run_until_complete_nesting(self): + @asyncio.coroutine + def coro1(): + yield + + @asyncio.coroutine + def coro2(): + self.assertTrue(self.loop.is_running()) + self.loop.run_until_complete(coro1()) + + self.assertRaises( + RuntimeError, self.loop.run_until_complete, coro2()) + + # Note: because of the default Windows timing granularity of + # 15.6 msec, we use fairly long sleep times here (~100 msec). + + def test_run_until_complete(self): + t0 = self.loop.time() + self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop)) + t1 = self.loop.time() + self.assertTrue(0.08 <= t1-t0 <= 0.8, t1-t0) + + def test_run_until_complete_stopped(self): + @asyncio.coroutine + def cb(): + self.loop.stop() + yield from asyncio.sleep(0.1, loop=self.loop) + task = cb() + self.assertRaises(RuntimeError, + self.loop.run_until_complete, task) + + def test_call_later(self): + results = [] + + def callback(arg): + results.append(arg) + self.loop.stop() + + self.loop.call_later(0.1, callback, 'hello world') + t0 = time.monotonic() + self.loop.run_forever() + t1 = time.monotonic() + self.assertEqual(results, ['hello world']) + self.assertTrue(0.08 <= t1-t0 <= 0.8, t1-t0) + + def test_call_soon(self): + results = [] + + def callback(arg1, arg2): + results.append((arg1, arg2)) + self.loop.stop() + + self.loop.call_soon(callback, 'hello', 'world') + self.loop.run_forever() + self.assertEqual(results, [('hello', 'world')]) + + def test_call_soon_threadsafe(self): + results = [] + lock = threading.Lock() + + def callback(arg): + results.append(arg) + if len(results) >= 2: + self.loop.stop() + + def run_in_thread(): + self.loop.call_soon_threadsafe(callback, 'hello') + lock.release() + + lock.acquire() + t = threading.Thread(target=run_in_thread) + t.start() + + with lock: + self.loop.call_soon(callback, 'world') + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello', 'world']) + + def test_call_soon_threadsafe_same_thread(self): + results = [] + + def callback(arg): + results.append(arg) + if len(results) >= 2: + self.loop.stop() + + self.loop.call_soon_threadsafe(callback, 'hello') + self.loop.call_soon(callback, 'world') + self.loop.run_forever() + self.assertEqual(results, ['hello', 'world']) + + def test_run_in_executor(self): + def run(arg): + return (arg, threading.get_ident()) + f2 = self.loop.run_in_executor(None, run, 'yo') + res, thread_id = self.loop.run_until_complete(f2) + self.assertEqual(res, 'yo') + self.assertNotEqual(thread_id, threading.get_ident()) + + def test_reader_callback(self): + r, w = test_utils.socketpair() + r.setblocking(False) + bytes_read = bytearray() + + def reader(): + try: + data = r.recv(1024) + except BlockingIOError: + # Spurious readiness notifications are possible + # at least on Linux -- see man select. + return + if data: + bytes_read.extend(data) + else: + self.assertTrue(self.loop.remove_reader(r.fileno())) + r.close() + + self.loop.add_reader(r.fileno(), reader) + self.loop.call_soon(w.send, b'abc') + test_utils.run_until(self.loop, lambda: len(bytes_read) >= 3) + self.loop.call_soon(w.send, b'def') + test_utils.run_until(self.loop, lambda: len(bytes_read) >= 6) + self.loop.call_soon(w.close) + self.loop.call_soon(self.loop.stop) + self.loop.run_forever() + self.assertEqual(bytes_read, b'abcdef') + + def test_writer_callback(self): + r, w = test_utils.socketpair() + w.setblocking(False) + + def writer(data): + w.send(data) + self.loop.stop() + + data = b'x' * 1024 + self.loop.add_writer(w.fileno(), writer, data) + self.loop.run_forever() + + self.assertTrue(self.loop.remove_writer(w.fileno())) + self.assertFalse(self.loop.remove_writer(w.fileno())) + + w.close() + read = r.recv(len(data) * 2) + r.close() + self.assertEqual(read, data) + + def _basetest_sock_client_ops(self, httpd, sock): + if not isinstance(self.loop, proactor_events.BaseProactorEventLoop): + # in debug mode, socket operations must fail + # if the socket is not in blocking mode + self.loop.set_debug(True) + sock.setblocking(True) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_connect(sock, httpd.address)) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_accept(sock)) + + # test in non-blocking mode + sock.setblocking(False) + self.loop.run_until_complete( + self.loop.sock_connect(sock, httpd.address)) + self.loop.run_until_complete( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + data = self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + # consume data + self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + sock.close() + self.assertTrue(data.startswith(b'HTTP/1.0 200 OK')) + + def test_sock_client_ops(self): + with test_utils.run_test_server() as httpd: + sock = socket.socket() + self._basetest_sock_client_ops(httpd, sock) + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_unix_sock_client_ops(self): + with test_utils.run_test_unix_server() as httpd: + sock = socket.socket(socket.AF_UNIX) + self._basetest_sock_client_ops(httpd, sock) + + def test_sock_client_fail(self): + # Make sure that we will get an unused port + address = None + try: + s = socket.socket() + s.bind(('127.0.0.1', 0)) + address = s.getsockname() + finally: + s.close() + + sock = socket.socket() + sock.setblocking(False) + with self.assertRaises(ConnectionRefusedError): + self.loop.run_until_complete( + self.loop.sock_connect(sock, address)) + sock.close() + + def test_sock_accept(self): + listener = socket.socket() + listener.setblocking(False) + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + client = socket.socket() + client.connect(listener.getsockname()) + + f = self.loop.sock_accept(listener) + conn, addr = self.loop.run_until_complete(f) + self.assertEqual(conn.gettimeout(), 0) + self.assertEqual(addr, client.getsockname()) + self.assertEqual(client.getpeername(), listener.getsockname()) + client.close() + conn.close() + listener.close() + + @unittest.skipUnless(hasattr(signal, 'SIGKILL'), 'No SIGKILL') + def test_add_signal_handler(self): + caught = 0 + + def my_handler(): + nonlocal caught + caught += 1 + + # Check error behavior first. + self.assertRaises( + TypeError, self.loop.add_signal_handler, 'boom', my_handler) + self.assertRaises( + TypeError, self.loop.remove_signal_handler, 'boom') + self.assertRaises( + ValueError, self.loop.add_signal_handler, signal.NSIG+1, + my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, signal.NSIG+1) + self.assertRaises( + ValueError, self.loop.add_signal_handler, 0, my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, 0) + self.assertRaises( + ValueError, self.loop.add_signal_handler, -1, my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, -1) + self.assertRaises( + RuntimeError, self.loop.add_signal_handler, signal.SIGKILL, + my_handler) + # Removing SIGKILL doesn't raise, since we don't call signal(). + self.assertFalse(self.loop.remove_signal_handler(signal.SIGKILL)) + # Now set a handler and handle it. + self.loop.add_signal_handler(signal.SIGINT, my_handler) + + os.kill(os.getpid(), signal.SIGINT) + test_utils.run_until(self.loop, lambda: caught) + + # Removing it should restore the default handler. + self.assertTrue(self.loop.remove_signal_handler(signal.SIGINT)) + self.assertEqual(signal.getsignal(signal.SIGINT), + signal.default_int_handler) + # Removing again returns False. + self.assertFalse(self.loop.remove_signal_handler(signal.SIGINT)) + + @unittest.skipUnless(hasattr(signal, 'SIGALRM'), 'No SIGALRM') + def test_signal_handling_while_selecting(self): + # Test with a signal actually arriving during a select() call. + caught = 0 + + def my_handler(): + nonlocal caught + caught += 1 + self.loop.stop() + + self.loop.add_signal_handler(signal.SIGALRM, my_handler) + + signal.setitimer(signal.ITIMER_REAL, 0.01, 0) # Send SIGALRM once. + self.loop.run_forever() + self.assertEqual(caught, 1) + + @unittest.skipUnless(hasattr(signal, 'SIGALRM'), 'No SIGALRM') + def test_signal_handling_args(self): + some_args = (42,) + caught = 0 + + def my_handler(*args): + nonlocal caught + caught += 1 + self.assertEqual(args, some_args) + + self.loop.add_signal_handler(signal.SIGALRM, my_handler, *some_args) + + signal.setitimer(signal.ITIMER_REAL, 0.1, 0) # Send SIGALRM once. + self.loop.call_later(0.5, self.loop.stop) + self.loop.run_forever() + self.assertEqual(caught, 1) + + def _basetest_create_connection(self, connection_fut, check_sockname=True): + tr, pr = self.loop.run_until_complete(connection_fut) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.assertIs(pr.transport, tr) + if check_sockname: + self.assertIsNotNone(tr.get_extra_info('sockname')) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + def test_create_connection(self): + with test_utils.run_test_server() as httpd: + conn_fut = self.loop.create_connection( + lambda: MyProto(loop=self.loop), *httpd.address) + self._basetest_create_connection(conn_fut) + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_unix_connection(self): + # Issue #20682: On Mac OS X Tiger, getsockname() returns a + # zero-length address for UNIX socket. + check_sockname = not osx_tiger() + + with test_utils.run_test_unix_server() as httpd: + conn_fut = self.loop.create_unix_connection( + lambda: MyProto(loop=self.loop), httpd.address) + self._basetest_create_connection(conn_fut, check_sockname) + + def test_create_connection_sock(self): + with test_utils.run_test_server() as httpd: + sock = None + infos = self.loop.run_until_complete( + self.loop.getaddrinfo( + *httpd.address, type=socket.SOCK_STREAM)) + for family, type, proto, cname, address in infos: + try: + sock = socket.socket(family=family, type=type, proto=proto) + sock.setblocking(False) + self.loop.run_until_complete( + self.loop.sock_connect(sock, address)) + except: + pass + else: + break + else: + assert False, 'Can not create socket.' + + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), sock=sock) + tr, pr = self.loop.run_until_complete(f) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + def check_ssl_extra_info(self, client, check_sockname=True, + peername=None, peercert={}): + if check_sockname: + self.assertIsNotNone(client.get_extra_info('sockname')) + if peername: + self.assertEqual(peername, + client.get_extra_info('peername')) + else: + self.assertIsNotNone(client.get_extra_info('peername')) + self.assertEqual(peercert, + client.get_extra_info('peercert')) + + # test SSL cipher + cipher = client.get_extra_info('cipher') + self.assertIsInstance(cipher, tuple) + self.assertEqual(len(cipher), 3, cipher) + self.assertIsInstance(cipher[0], str) + self.assertIsInstance(cipher[1], str) + self.assertIsInstance(cipher[2], int) + + # test SSL object + sslobj = client.get_extra_info('ssl_object') + self.assertIsNotNone(sslobj) + self.assertEqual(sslobj.compression(), + client.get_extra_info('compression')) + self.assertEqual(sslobj.cipher(), + client.get_extra_info('cipher')) + self.assertEqual(sslobj.getpeercert(), + client.get_extra_info('peercert')) + self.assertEqual(sslobj.compression(), + client.get_extra_info('compression')) + + def _basetest_create_ssl_connection(self, connection_fut, + check_sockname=True, + peername=None): + tr, pr = self.loop.run_until_complete(connection_fut) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.assertTrue('ssl' in tr.__class__.__name__.lower()) + self.check_ssl_extra_info(tr, check_sockname, peername) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + def _test_create_ssl_connection(self, httpd, create_connection, + check_sockname=True, peername=None): + conn_fut = create_connection(ssl=test_utils.dummy_ssl_context()) + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + + # ssl.Purpose was introduced in Python 3.4 + if hasattr(ssl, 'Purpose'): + def _dummy_ssl_create_context(purpose=ssl.Purpose.SERVER_AUTH, *, + cafile=None, capath=None, + cadata=None): + """ + A ssl.create_default_context() replacement that doesn't enable + cert validation. + """ + self.assertEqual(purpose, ssl.Purpose.SERVER_AUTH) + return test_utils.dummy_ssl_context() + + # With ssl=True, ssl.create_default_context() should be called + with mock.patch('ssl.create_default_context', + side_effect=_dummy_ssl_create_context) as m: + conn_fut = create_connection(ssl=True) + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + self.assertEqual(m.call_count, 1) + + # With the real ssl.create_default_context(), certificate + # validation will fail + with self.assertRaises(ssl.SSLError) as cm: + conn_fut = create_connection(ssl=True) + # Ignore the "SSL handshake failed" log in debug mode + with test_utils.disable_logger(): + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + + self.assertEqual(cm.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_ssl_connection(self): + with test_utils.run_test_server(use_ssl=True) as httpd: + create_connection = functools.partial( + self.loop.create_connection, + lambda: MyProto(loop=self.loop), + *httpd.address) + self._test_create_ssl_connection(httpd, create_connection, + peername=httpd.address) + + def test_legacy_create_ssl_connection(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_ssl_connection() + + @unittest.skipIf(ssl is None, 'No ssl module') + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_ssl_unix_connection(self): + # Issue #20682: On Mac OS X Tiger, getsockname() returns a + # zero-length address for UNIX socket. + check_sockname = not osx_tiger() + + with test_utils.run_test_unix_server(use_ssl=True) as httpd: + create_connection = functools.partial( + self.loop.create_unix_connection, + lambda: MyProto(loop=self.loop), httpd.address, + server_hostname='127.0.0.1') + + self._test_create_ssl_connection(httpd, create_connection, + check_sockname, + peername=httpd.address) + + def test_legacy_create_ssl_unix_connection(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_ssl_unix_connection() + + def test_create_connection_local_addr(self): + with test_utils.run_test_server() as httpd: + port = support.find_unused_port() + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + *httpd.address, local_addr=(httpd.address[0], port)) + tr, pr = self.loop.run_until_complete(f) + expected = pr.transport.get_extra_info('sockname')[1] + self.assertEqual(port, expected) + tr.close() + + def test_create_connection_local_addr_in_use(self): + with test_utils.run_test_server() as httpd: + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + *httpd.address, local_addr=httpd.address) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(f) + self.assertEqual(cm.exception.errno, errno.EADDRINUSE) + self.assertIn(str(httpd.address), cm.exception.strerror) + + def test_connect_accepted_socket(self, server_ssl=None, client_ssl=None): + loop = self.loop + + class MyProto(MyBaseProto): + + def connection_lost(self, exc): + super().connection_lost(exc) + loop.call_soon(loop.stop) + + def data_received(self, data): + super().data_received(data) + self.transport.write(expected_response) + + lsock = socket.socket() + lsock.bind(('127.0.0.1', 0)) + lsock.listen(1) + addr = lsock.getsockname() + + message = b'test data' + response = None + expected_response = b'roger' + + def client(): + nonlocal response + try: + csock = socket.socket() + if client_ssl is not None: + csock = client_ssl.wrap_socket(csock) + csock.connect(addr) + csock.sendall(message) + response = csock.recv(99) + csock.close() + except Exception as exc: + print( + "Failure in client thread in test_connect_accepted_socket", + exc) + + thread = threading.Thread(target=client, daemon=True) + thread.start() + + conn, _ = lsock.accept() + proto = MyProto(loop=loop) + proto.loop = loop + f = loop.create_task( + loop.connect_accepted_socket( + (lambda : proto), conn, ssl=server_ssl)) + loop.run_forever() + proto.transport.close() + lsock.close() + + thread.join(1) + self.assertFalse(thread.is_alive()) + self.assertEqual(proto.state, 'CLOSED') + self.assertEqual(proto.nbytes, len(message)) + self.assertEqual(response, expected_response) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_ssl_connect_accepted_socket(self): + if (sys.platform == 'win32' and + sys.version_info < (3, 5) and + isinstance(self.loop, proactor_events.BaseProactorEventLoop) + ): + raise unittest.SkipTest( + 'SSL not supported with proactor event loops before Python 3.5' + ) + + server_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + server_context.load_cert_chain(ONLYCERT, ONLYKEY) + if hasattr(server_context, 'check_hostname'): + server_context.check_hostname = False + server_context.verify_mode = ssl.CERT_NONE + + client_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + if hasattr(server_context, 'check_hostname'): + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + + self.test_connect_accepted_socket(server_context, client_context) + + @mock.patch('asyncio.base_events.socket') + def create_server_multiple_hosts(self, family, hosts, mock_sock): + @asyncio.coroutine + def getaddrinfo(host, port, *args, **kw): + if family == socket.AF_INET: + return [(family, socket.SOCK_STREAM, 6, '', (host, port))] + else: + return [(family, socket.SOCK_STREAM, 6, '', (host, port, 0, 0))] + + def getaddrinfo_task(*args, **kwds): + return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop) + + unique_hosts = set(hosts) + + if family == socket.AF_INET: + mock_sock.socket().getsockbyname.side_effect = [ + (host, 80) for host in unique_hosts] + else: + mock_sock.socket().getsockbyname.side_effect = [ + (host, 80, 0, 0) for host in unique_hosts] + self.loop.getaddrinfo = getaddrinfo_task + self.loop._start_serving = mock.Mock() + self.loop._stop_serving = mock.Mock() + f = self.loop.create_server(lambda: MyProto(self.loop), hosts, 80) + server = self.loop.run_until_complete(f) + self.addCleanup(server.close) + server_hosts = {sock.getsockbyname()[0] for sock in server.sockets} + self.assertEqual(server_hosts, unique_hosts) + + def test_create_server_multiple_hosts_ipv4(self): + self.create_server_multiple_hosts(socket.AF_INET, + ['1.2.3.4', '5.6.7.8', '1.2.3.4']) + + def test_create_server_multiple_hosts_ipv6(self): + self.create_server_multiple_hosts(socket.AF_INET6, + ['::1', '::2', '::1']) + + def test_create_server(self): + proto = MyProto(self.loop) + f = self.loop.create_server(lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.sendall(b'xxx') + + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('sockname')) + self.assertEqual('127.0.0.1', + proto.transport.get_extra_info('peername')[0]) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # close server + server.close() + + @unittest.skipUnless(hasattr(socket, 'SO_REUSEPORT'), 'No SO_REUSEPORT') + def test_create_server_reuse_port(self): + proto = MyProto(self.loop) + f = self.loop.create_server( + lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + server.close() + + test_utils.run_briefly(self.loop) + + proto = MyProto(self.loop) + f = self.loop.create_server( + lambda: proto, '0.0.0.0', 0, reuse_port=True) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + server.close() + + def _make_unix_server(self, factory, **kwargs): + path = test_utils.gen_unix_socket_path() + self.addCleanup(lambda: os.path.exists(path) and os.unlink(path)) + + f = self.loop.create_unix_server(factory, path, **kwargs) + server = self.loop.run_until_complete(f) + + return server, path + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_unix_server(self): + proto = MyProto(loop=self.loop) + server, path = self._make_unix_server(lambda: proto) + self.assertEqual(len(server.sockets), 1) + + client = socket.socket(socket.AF_UNIX) + client.connect(path) + client.sendall(b'xxx') + + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # close server + server.close() + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_unix_server_path_socket_error(self): + proto = MyProto(loop=self.loop) + sock = socket.socket() + with sock: + f = self.loop.create_unix_server(lambda: proto, '/test', sock=sock) + with self.assertRaisesRegex(ValueError, + 'path and sock can not be specified ' + 'at the same time'): + self.loop.run_until_complete(f) + + def _create_ssl_context(self, certfile, keyfile=None): + sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.load_cert_chain(certfile, keyfile) + return sslcontext + + def _make_ssl_server(self, factory, certfile, keyfile=None): + sslcontext = self._create_ssl_context(certfile, keyfile) + + f = self.loop.create_server(factory, '127.0.0.1', 0, ssl=sslcontext) + server = self.loop.run_until_complete(f) + + sock = server.sockets[0] + host, port = sock.getsockname() + self.assertEqual(host, '127.0.0.1') + return server, host, port + + def _make_ssl_unix_server(self, factory, certfile, keyfile=None): + sslcontext = self._create_ssl_context(certfile, keyfile) + return self._make_unix_server(factory, ssl=sslcontext) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, ONLYCERT, ONLYKEY) + + f_c = self.loop.create_connection(MyBaseProto, host, port, + ssl=test_utils.dummy_ssl_context()) + client, pr = self.loop.run_until_complete(f_c) + + client.write(b'xxx') + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # extra info is available + self.check_ssl_extra_info(client, peername=(host, port)) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # stop serving + server.close() + + def test_legacy_create_server_ssl(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_server_ssl() + + @unittest.skipIf(ssl is None, 'No ssl module') + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_unix_server_ssl(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, ONLYCERT, ONLYKEY) + + f_c = self.loop.create_unix_connection( + MyBaseProto, path, ssl=test_utils.dummy_ssl_context(), + server_hostname='') + + client, pr = self.loop.run_until_complete(f_c) + + client.write(b'xxx') + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # stop serving + server.close() + + def test_legacy_create_unix_server_ssl(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_unix_server_ssl() + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_verify_failed(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + + # no CA loaded + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + '(?i)certificate.verify.failed'): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) + + # close connection + self.assertIsNone(proto.transport) + server.close() + + def test_legacy_create_server_ssl_verify_failed(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_server_ssl_verify_failed() + + @unittest.skipIf(ssl is None, 'No ssl module') + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_unix_server_ssl_verify_failed(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # no CA loaded + f_c = self.loop.create_unix_connection(MyProto, path, + ssl=sslcontext_client, + server_hostname='invalid') + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + '(?i)certificate.verify.failed'): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) + + # close connection + self.assertIsNone(proto.transport) + server.close() + + + def test_legacy_create_unix_server_ssl_verify_failed(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_unix_server_ssl_verify_failed() + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_match_failed(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations( + cafile=SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # incorrect server_hostname + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex( + ssl.CertificateError, + "hostname '127.0.0.1' doesn't match 'localhost'"): + self.loop.run_until_complete(f_c) + + # close connection + proto.transport.close() + server.close() + + def test_legacy_create_server_ssl_match_failed(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_server_ssl_match_failed() + + @unittest.skipIf(ssl is None, 'No ssl module') + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_unix_server_ssl_verified(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations(cafile=SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # Connection succeeds with correct CA and server hostname. + f_c = self.loop.create_unix_connection(MyProto, path, + ssl=sslcontext_client, + server_hostname='localhost') + client, pr = self.loop.run_until_complete(f_c) + + # close connection + proto.transport.close() + client.close() + server.close() + self.loop.run_until_complete(proto.done) + + def test_legacy_create_unix_server_ssl_verified(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_unix_server_ssl_verified() + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_verified(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations(cafile=SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # Connection succeeds with correct CA and server hostname. + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client, + server_hostname='localhost') + client, pr = self.loop.run_until_complete(f_c) + + # extra info is available + self.check_ssl_extra_info(client,peername=(host, port), + peercert=PEERCERT) + + # close connection + proto.transport.close() + client.close() + server.close() + self.loop.run_until_complete(proto.done) + + def test_legacy_create_server_ssl_verified(self): + with test_utils.force_legacy_ssl_support(): + self.test_create_server_ssl_verified() + + def test_create_server_sock(self): + proto = asyncio.Future(loop=self.loop) + + class TestMyProto(MyProto): + def connection_made(self, transport): + super().connection_made(transport) + proto.set_result(self) + + sock_ob = socket.socket(type=socket.SOCK_STREAM) + sock_ob.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock_ob.bind(('0.0.0.0', 0)) + + f = self.loop.create_server(TestMyProto, sock=sock_ob) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + self.assertIs(sock, sock_ob) + + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + client.close() + server.close() + + def test_create_server_addr_in_use(self): + sock_ob = socket.socket(type=socket.SOCK_STREAM) + sock_ob.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock_ob.bind(('0.0.0.0', 0)) + + f = self.loop.create_server(MyProto, sock=sock_ob) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + host, port = sock.getsockname() + + f = self.loop.create_server(MyProto, host=host, port=port) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(f) + self.assertEqual(cm.exception.errno, errno.EADDRINUSE) + + server.close() + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_server_dual_stack(self): + f_proto = asyncio.Future(loop=self.loop) + + class TestMyProto(MyProto): + def connection_made(self, transport): + super().connection_made(transport) + f_proto.set_result(self) + + try_count = 0 + while True: + try: + port = support.find_unused_port() + f = self.loop.create_server(TestMyProto, host=None, port=port) + server = self.loop.run_until_complete(f) + except OSError as ex: + if ex.errno == errno.EADDRINUSE: + try_count += 1 + self.assertGreaterEqual(5, try_count) + continue + else: + raise + else: + break + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + proto = self.loop.run_until_complete(f_proto) + proto.transport.close() + client.close() + + f_proto = asyncio.Future(loop=self.loop) + client = socket.socket(socket.AF_INET6) + client.connect(('::1', port)) + client.send(b'xxx') + proto = self.loop.run_until_complete(f_proto) + proto.transport.close() + client.close() + + server.close() + + def test_server_close(self): + f = self.loop.create_server(MyProto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + host, port = sock.getsockname() + + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + client.close() + + server.close() + + client = socket.socket() + self.assertRaises( + ConnectionRefusedError, client.connect, ('127.0.0.1', port)) + client.close() + + def test_create_datagram_endpoint(self): + class TestMyDatagramProto(MyDatagramProto): + def __init__(inner_self): + super().__init__(loop=self.loop) + + def datagram_received(self, data, addr): + super().datagram_received(data, addr) + self.transport.sendto(b'resp:'+data, addr) + + coro = self.loop.create_datagram_endpoint( + TestMyDatagramProto, local_addr=('127.0.0.1', 0)) + s_transport, server = self.loop.run_until_complete(coro) + host, port = s_transport.get_extra_info('sockname') + + self.assertIsInstance(s_transport, asyncio.Transport) + self.assertIsInstance(server, TestMyDatagramProto) + self.assertEqual('INITIALIZED', server.state) + self.assertIs(server.transport, s_transport) + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + remote_addr=(host, port)) + transport, client = self.loop.run_until_complete(coro) + + self.assertIsInstance(transport, asyncio.Transport) + self.assertIsInstance(client, MyDatagramProto) + self.assertEqual('INITIALIZED', client.state) + self.assertIs(client.transport, transport) + + transport.sendto(b'xxx') + test_utils.run_until(self.loop, lambda: server.nbytes) + self.assertEqual(3, server.nbytes) + test_utils.run_until(self.loop, lambda: client.nbytes) + + # received + self.assertEqual(8, client.nbytes) + + # extra info is available + self.assertIsNotNone(transport.get_extra_info('sockname')) + + # close connection + transport.close() + self.loop.run_until_complete(client.done) + self.assertEqual('CLOSED', client.state) + server.transport.close() + + def test_create_datagram_endpoint_sock(self): + sock = None + local_address = ('127.0.0.1', 0) + infos = self.loop.run_until_complete( + self.loop.getaddrinfo( + *local_address, type=socket.SOCK_DGRAM)) + for family, type, proto, cname, address in infos: + try: + sock = socket.socket(family=family, type=type, proto=proto) + sock.setblocking(False) + sock.bind(address) + except: + pass + else: + break + else: + assert False, 'Can not create socket.' + + f = self.loop.create_connection( + lambda: MyDatagramProto(loop=self.loop), sock=sock) + tr, pr = self.loop.run_until_complete(f) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, MyDatagramProto) + tr.close() + self.loop.run_until_complete(pr.done) + + def test_internal_fds(self): + loop = self.create_event_loop() + if not isinstance(loop, selector_events.BaseSelectorEventLoop): + loop.close() + self.skipTest('loop is not a BaseSelectorEventLoop') + + self.assertEqual(1, loop._internal_fds) + loop.close() + self.assertEqual(0, loop._internal_fds) + self.assertIsNone(loop._csock) + self.assertIsNone(loop._ssock) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_read_pipe(self): + proto = MyReadPipeProto(loop=self.loop) + + rpipe, wpipe = os.pipe() + pipeobj = io.open(rpipe, 'rb', 1024) + + @asyncio.coroutine + def connect(): + t, p = yield from self.loop.connect_read_pipe( + lambda: proto, pipeobj) + self.assertIs(p, proto) + self.assertIs(t, proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(0, proto.nbytes) + + self.loop.run_until_complete(connect()) + + os.write(wpipe, b'1') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 1) + self.assertEqual(1, proto.nbytes) + + os.write(wpipe, b'2345') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 5) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(5, proto.nbytes) + + os.close(wpipe) + self.loop.run_until_complete(proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_unclosed_pipe_transport(self): + # This test reproduces the issue #314 on GitHub + loop = self.create_event_loop() + read_proto = MyReadPipeProto(loop=loop) + write_proto = MyWritePipeProto(loop=loop) + + rpipe, wpipe = os.pipe() + rpipeobj = io.open(rpipe, 'rb', 1024) + wpipeobj = io.open(wpipe, 'w', 1024) + + @asyncio.coroutine + def connect(): + read_transport, _ = yield from loop.connect_read_pipe( + lambda: read_proto, rpipeobj) + write_transport, _ = yield from loop.connect_write_pipe( + lambda: write_proto, wpipeobj) + return read_transport, write_transport + + # Run and close the loop without closing the transports + read_transport, write_transport = loop.run_until_complete(connect()) + loop.close() + + # These 'repr' calls used to raise an AttributeError + # See Issue #314 on GitHub + self.assertIn('open', repr(read_transport)) + self.assertIn('open', repr(write_transport)) + + # Clean up (avoid ResourceWarning) + rpipeobj.close() + wpipeobj.close() + read_transport._pipe = None + write_transport._pipe = None + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + # select, poll and kqueue don't support character devices (PTY) on Mac OS X + # older than 10.6 (Snow Leopard) + @support.requires_mac_ver(10, 6) + # Issue #20495: The test hangs on FreeBSD 7.2 but pass on FreeBSD 9 + @support.requires_freebsd_version(8) + def test_read_pty_output(self): + proto = MyReadPipeProto(loop=self.loop) + + master, slave = os.openpty() + master_read_obj = io.open(master, 'rb', 0) + + @asyncio.coroutine + def connect(): + t, p = yield from self.loop.connect_read_pipe(lambda: proto, + master_read_obj) + self.assertIs(p, proto) + self.assertIs(t, proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(0, proto.nbytes) + + self.loop.run_until_complete(connect()) + + os.write(slave, b'1') + test_utils.run_until(self.loop, lambda: proto.nbytes) + self.assertEqual(1, proto.nbytes) + + os.write(slave, b'2345') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 5) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(5, proto.nbytes) + + os.close(slave) + self.loop.run_until_complete(proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_write_pipe(self): + rpipe, wpipe = os.pipe() + pipeobj = io.open(wpipe, 'wb', 1024) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, pipeobj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + + data = bytearray() + def reader(data): + chunk = os.read(rpipe, 1024) + data += chunk + return len(data) + + test_utils.run_until(self.loop, lambda: reader(data) >= 1) + self.assertEqual(b'1', data) + + transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5) + self.assertEqual(b'12345', data) + self.assertEqual('CONNECTED', proto.state) + + os.close(rpipe) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_write_pipe_disconnect_on_close(self): + rsock, wsock = test_utils.socketpair() + rsock.setblocking(False) + pipeobj = io.open(wsock.detach(), 'wb', 1024) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, pipeobj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + data = self.loop.run_until_complete(self.loop.sock_recv(rsock, 1024)) + self.assertEqual(b'1', data) + + rsock.close() + + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + # select, poll and kqueue don't support character devices (PTY) on Mac OS X + # older than 10.6 (Snow Leopard) + @support.requires_mac_ver(10, 6) + def test_write_pty(self): + master, slave = os.openpty() + slave_write_obj = io.open(slave, 'wb', 0) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, slave_write_obj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + + data = bytearray() + def reader(data): + chunk = os.read(master, 1024) + data += chunk + return len(data) + + test_utils.run_until(self.loop, lambda: reader(data) >= 1, + timeout=10) + self.assertEqual(b'1', data) + + transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5, + timeout=10) + self.assertEqual(b'12345', data) + self.assertEqual('CONNECTED', proto.state) + + os.close(master) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + # select, poll and kqueue don't support character devices (PTY) on Mac OS X + # older than 10.6 (Snow Leopard) + @support.requires_mac_ver(10, 6) + def test_bidirectional_pty(self): + master, read_slave = os.openpty() + write_slave = os.dup(read_slave) + tty.setraw(read_slave) + + slave_read_obj = io.open(read_slave, 'rb', 0) + read_proto = MyReadPipeProto(loop=self.loop) + read_connect = self.loop.connect_read_pipe(lambda: read_proto, + slave_read_obj) + read_transport, p = self.loop.run_until_complete(read_connect) + self.assertIs(p, read_proto) + self.assertIs(read_transport, read_proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(0, read_proto.nbytes) + + + slave_write_obj = io.open(write_slave, 'wb', 0) + write_proto = MyWritePipeProto(loop=self.loop) + write_connect = self.loop.connect_write_pipe(lambda: write_proto, + slave_write_obj) + write_transport, p = self.loop.run_until_complete(write_connect) + self.assertIs(p, write_proto) + self.assertIs(write_transport, write_proto.transport) + self.assertEqual('CONNECTED', write_proto.state) + + data = bytearray() + def reader(data): + chunk = os.read(master, 1024) + data += chunk + return len(data) + + write_transport.write(b'1') + test_utils.run_until(self.loop, lambda: reader(data) >= 1, timeout=10) + self.assertEqual(b'1', data) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual('CONNECTED', write_proto.state) + + os.write(master, b'a') + test_utils.run_until(self.loop, lambda: read_proto.nbytes >= 1, + timeout=10) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(1, read_proto.nbytes) + self.assertEqual('CONNECTED', write_proto.state) + + write_transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5, timeout=10) + self.assertEqual(b'12345', data) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual('CONNECTED', write_proto.state) + + os.write(master, b'bcde') + test_utils.run_until(self.loop, lambda: read_proto.nbytes >= 5, + timeout=10) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(5, read_proto.nbytes) + self.assertEqual('CONNECTED', write_proto.state) + + os.close(master) + + read_transport.close() + self.loop.run_until_complete(read_proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], read_proto.state) + + write_transport.close() + self.loop.run_until_complete(write_proto.done) + self.assertEqual('CLOSED', write_proto.state) + + def test_prompt_cancellation(self): + r, w = test_utils.socketpair() + r.setblocking(False) + f = self.loop.sock_recv(r, 1) + ov = getattr(f, 'ov', None) + if ov is not None: + self.assertTrue(ov.pending) + + @asyncio.coroutine + def main(): + try: + self.loop.call_soon(f.cancel) + yield from f + except asyncio.CancelledError: + res = 'cancelled' + else: + res = None + finally: + self.loop.stop() + return res + + start = time.monotonic() + t = asyncio.Task(main(), loop=self.loop) + self.loop.run_forever() + elapsed = time.monotonic() - start + + self.assertLess(elapsed, 0.1) + self.assertEqual(t.result(), 'cancelled') + self.assertRaises(asyncio.CancelledError, f.result) + if ov is not None: + self.assertFalse(ov.pending) + self.loop._stop_serving(r) + + r.close() + w.close() + + def test_timeout_rounding(self): + def _run_once(): + self.loop._run_once_counter += 1 + orig_run_once() + + orig_run_once = self.loop._run_once + self.loop._run_once_counter = 0 + self.loop._run_once = _run_once + + @asyncio.coroutine + def wait(): + loop = self.loop + yield from asyncio.sleep(1e-2, loop=loop) + yield from asyncio.sleep(1e-4, loop=loop) + yield from asyncio.sleep(1e-6, loop=loop) + yield from asyncio.sleep(1e-8, loop=loop) + yield from asyncio.sleep(1e-10, loop=loop) + + self.loop.run_until_complete(wait()) + # The ideal number of call is 12, but on some platforms, the selector + # may sleep at little bit less than timeout depending on the resolution + # of the clock used by the kernel. Tolerate a few useless calls on + # these platforms. + self.assertLessEqual(self.loop._run_once_counter, 20, + {'clock_resolution': self.loop._clock_resolution, + 'selector': self.loop._selector.__class__.__name__}) + + def test_remove_fds_after_closing(self): + loop = self.create_event_loop() + callback = lambda: None + r, w = test_utils.socketpair() + self.addCleanup(r.close) + self.addCleanup(w.close) + loop.add_reader(r, callback) + loop.add_writer(w, callback) + loop.close() + self.assertFalse(loop.remove_reader(r)) + self.assertFalse(loop.remove_writer(w)) + + def test_add_fds_after_closing(self): + loop = self.create_event_loop() + callback = lambda: None + r, w = test_utils.socketpair() + self.addCleanup(r.close) + self.addCleanup(w.close) + loop.close() + with self.assertRaises(RuntimeError): + loop.add_reader(r, callback) + with self.assertRaises(RuntimeError): + loop.add_writer(w, callback) + + def test_close_running_event_loop(self): + @asyncio.coroutine + def close_loop(loop): + self.loop.close() + + coro = close_loop(self.loop) + with self.assertRaises(RuntimeError): + self.loop.run_until_complete(coro) + + def test_close(self): + self.loop.close() + + @asyncio.coroutine + def test(): + pass + + func = lambda: False + coro = test() + self.addCleanup(coro.close) + + # operation blocked when the loop is closed + with self.assertRaises(RuntimeError): + self.loop.run_forever() + with self.assertRaises(RuntimeError): + fut = asyncio.Future(loop=self.loop) + self.loop.run_until_complete(fut) + with self.assertRaises(RuntimeError): + self.loop.call_soon(func) + with self.assertRaises(RuntimeError): + self.loop.call_soon_threadsafe(func) + with self.assertRaises(RuntimeError): + self.loop.call_later(1.0, func) + with self.assertRaises(RuntimeError): + self.loop.call_at(self.loop.time() + .0, func) + with self.assertRaises(RuntimeError): + self.loop.run_in_executor(None, func) + with self.assertRaises(RuntimeError): + self.loop.create_task(coro) + with self.assertRaises(RuntimeError): + self.loop.add_signal_handler(signal.SIGTERM, func) + + +class SubprocessTestsMixin: + + def check_terminated(self, returncode): + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGTERM, returncode) + + def check_killed(self, returncode): + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + + def test_subprocess_exec(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'Python The Winner') + self.loop.run_until_complete(proto.got_data[1].wait()) + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + self.assertEqual(b'Python The Winner', proto.data[1]) + + def test_subprocess_interactive(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'Python ') + self.loop.run_until_complete(proto.got_data[1].wait()) + proto.got_data[1].clear() + self.assertEqual(b'Python ', proto.data[1]) + + stdin.write(b'The Winner') + self.loop.run_until_complete(proto.got_data[1].wait()) + self.assertEqual(b'Python The Winner', proto.data[1]) + + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + + def test_subprocess_shell(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'echo Python') + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.get_pipe_transport(0).close() + self.loop.run_until_complete(proto.completed) + self.assertEqual(0, proto.returncode) + self.assertTrue(all(f.done() for f in proto.disconnects.values())) + self.assertEqual(proto.data[1].rstrip(b'\r\n'), b'Python') + self.assertEqual(proto.data[2], b'') + transp.close() + + def test_subprocess_exitcode(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + transp.close() + + def test_subprocess_close_after_finish(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.assertIsNone(transp.get_pipe_transport(0)) + self.assertIsNone(transp.get_pipe_transport(1)) + self.assertIsNone(transp.get_pipe_transport(2)) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + self.assertIsNone(transp.close()) + + def test_subprocess_kill(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.kill() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + transp.close() + + def test_subprocess_terminate(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.terminate() + self.loop.run_until_complete(proto.completed) + self.check_terminated(proto.returncode) + transp.close() + + @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + def test_subprocess_send_signal(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.send_signal(signal.SIGHUP) + self.loop.run_until_complete(proto.completed) + self.assertEqual(-signal.SIGHUP, proto.returncode) + transp.close() + + def test_subprocess_stderr(self): + prog = os.path.join(os.path.dirname(__file__), 'echo2.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'test') + + self.loop.run_until_complete(proto.completed) + + transp.close() + self.assertEqual(b'OUT:test', proto.data[1]) + self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2]) + self.assertEqual(0, proto.returncode) + + def test_subprocess_stderr_redirect_to_stdout(self): + prog = os.path.join(os.path.dirname(__file__), 'echo2.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog, stderr=subprocess.STDOUT) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + self.assertIsNotNone(transp.get_pipe_transport(1)) + self.assertIsNone(transp.get_pipe_transport(2)) + + stdin.write(b'test') + self.loop.run_until_complete(proto.completed) + self.assertTrue(proto.data[1].startswith(b'OUT:testERR:test'), + proto.data[1]) + self.assertEqual(b'', proto.data[2]) + + transp.close() + self.assertEqual(0, proto.returncode) + + def test_subprocess_close_client_stream(self): + prog = os.path.join(os.path.dirname(__file__), 'echo3.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + stdout = transp.get_pipe_transport(1) + stdin.write(b'test') + self.loop.run_until_complete(proto.got_data[1].wait()) + self.assertEqual(b'OUT:test', proto.data[1]) + + stdout.close() + self.loop.run_until_complete(proto.disconnects[1]) + stdin.write(b'xxx') + self.loop.run_until_complete(proto.got_data[2].wait()) + if sys.platform != 'win32': + self.assertEqual(b'ERR:BrokenPipeError', proto.data[2]) + else: + # After closing the read-end of a pipe, writing to the + # write-end using os.write() fails with errno==EINVAL and + # GetLastError()==ERROR_INVALID_NAME on Windows!?! (Using + # WriteFile() we get ERROR_BROKEN_PIPE as expected.) + self.assertEqual(b'ERR:OSError', proto.data[2]) + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + + def test_subprocess_wait_no_same_group(self): + # start the new process in a new session + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None, + start_new_session=True) + _, proto = yield self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + + def test_subprocess_exec_invalid_args(self): + @asyncio.coroutine + def connect(**kwds): + yield from self.loop.subprocess_exec( + asyncio.SubprocessProtocol, + 'pwd', **kwds) + + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(universal_newlines=True)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(bufsize=4096)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(shell=True)) + + def test_subprocess_shell_invalid_args(self): + @asyncio.coroutine + def connect(cmd=None, **kwds): + if not cmd: + cmd = 'pwd' + yield from self.loop.subprocess_shell( + asyncio.SubprocessProtocol, + cmd, **kwds) + + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(['ls', '-l'])) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(universal_newlines=True)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(bufsize=4096)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(shell=False)) + + +if sys.platform == 'win32': + + class SelectEventLoopTests(EventLoopTestsMixin, test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop() + + class ProactorEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.ProactorEventLoop() + + if not sslproto._is_sslproto_available(): + def test_create_ssl_connection(self): + raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") + + def test_create_server_ssl(self): + raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") + + def test_create_server_ssl_verify_failed(self): + raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") + + def test_create_server_ssl_match_failed(self): + raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") + + def test_create_server_ssl_verified(self): + raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") + + def test_legacy_create_ssl_connection(self): + raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") + + def test_legacy_create_server_ssl(self): + raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") + + def test_legacy_create_server_ssl_verify_failed(self): + raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") + + def test_legacy_create_server_ssl_match_failed(self): + raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") + + def test_legacy_create_server_ssl_verified(self): + raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") + + def test_reader_callback(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") + + def test_reader_callback_cancel(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") + + def test_writer_callback(self): + raise unittest.SkipTest("IocpEventLoop does not have add_writer()") + + def test_writer_callback_cancel(self): + raise unittest.SkipTest("IocpEventLoop does not have add_writer()") + + def test_create_datagram_endpoint(self): + raise unittest.SkipTest( + "IocpEventLoop does not have create_datagram_endpoint()") + + def test_remove_fds_after_closing(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") +else: + from asyncio import selectors + + class UnixEventLoopTestsMixin(EventLoopTestsMixin): + def setUp(self): + super().setUp() + watcher = asyncio.SafeChildWatcher() + watcher.attach_loop(self.loop) + asyncio.set_child_watcher(watcher) + + def tearDown(self): + asyncio.set_child_watcher(None) + super().tearDown() + + if hasattr(selectors, 'KqueueSelector'): + class KqueueEventLoopTests(UnixEventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop( + selectors.KqueueSelector()) + + # kqueue doesn't support character devices (PTY) on Mac OS X older + # than 10.9 (Maverick) + @support.requires_mac_ver(10, 9) + # Issue #20667: KqueueEventLoopTests.test_read_pty_output() + # hangs on OpenBSD 5.5 + @unittest.skipIf(sys.platform.startswith('openbsd'), + 'test hangs on OpenBSD') + def test_read_pty_output(self): + super().test_read_pty_output() + + # kqueue doesn't support character devices (PTY) on Mac OS X older + # than 10.9 (Maverick) + @support.requires_mac_ver(10, 9) + def test_write_pty(self): + super().test_write_pty() + + if hasattr(selectors, 'EpollSelector'): + class EPollEventLoopTests(UnixEventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.EpollSelector()) + + if hasattr(selectors, 'PollSelector'): + class PollEventLoopTests(UnixEventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.PollSelector()) + + # Should always exist. + class SelectEventLoopTests(UnixEventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +def noop(*args, **kwargs): + pass + + +class HandleTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = mock.Mock() + self.loop.get_debug.return_value = True + + def test_handle(self): + def callback(*args): + return args + + args = () + h = asyncio.Handle(callback, args, self.loop) + self.assertIs(h._callback, callback) + self.assertIs(h._args, args) + self.assertFalse(h._cancelled) + + h.cancel() + self.assertTrue(h._cancelled) + + def test_callback_with_exception(self): + def callback(): + raise ValueError() + + self.loop = mock.Mock() + self.loop.call_exception_handler = mock.Mock() + + h = asyncio.Handle(callback, (), self.loop) + h._run() + + self.loop.call_exception_handler.assert_called_with({ + 'message': test_utils.MockPattern('Exception in callback.*'), + 'exception': mock.ANY, + 'handle': h, + 'source_traceback': h._source_traceback, + }) + + def test_handle_weakref(self): + wd = weakref.WeakValueDictionary() + h = asyncio.Handle(lambda: None, (), self.loop) + wd['h'] = h # Would fail without __weakref__ slot. + + def test_handle_repr(self): + self.loop.get_debug.return_value = False + + # simple function + h = asyncio.Handle(noop, (1, 2), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno)) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '') + + # decorated function + cb = asyncio.coroutine(noop) + h = asyncio.Handle(cb, (), self.loop) + self.assertEqual(repr(h), + '' + % (filename, lineno)) + + # partial function + cb = functools.partial(noop, 1, 2) + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno)) + self.assertRegex(repr(h), regex) + + # partial method + if sys.version_info >= (3, 4): + method = HandleTests.test_handle_repr + cb = functools.partialmethod(method) + filename, lineno = test_utils.get_function_source(method) + h = asyncio.Handle(cb, (), self.loop) + + cb_regex = r'' + cb_regex = (r'functools.partialmethod\(%s, , \)\(\)' % cb_regex) + regex = (r'^$' + % (cb_regex, re.escape(filename), lineno)) + self.assertRegex(repr(h), regex) + + def test_handle_repr_debug(self): + self.loop.get_debug.return_value = True + + # simple function + create_filename = __file__ + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(noop, (1, 2), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # cancelled handle + h.cancel() + self.assertEqual( + repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # double cancellation won't overwrite _repr + h.cancel() + self.assertEqual( + repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + def test_handle_source_traceback(self): + loop = asyncio.get_event_loop_policy().new_event_loop() + loop.set_debug(True) + self.set_event_loop(loop) + + def check_source_traceback(h): + lineno = sys._getframe(1).f_lineno - 1 + self.assertIsInstance(h._source_traceback, list) + self.assertEqual(h._source_traceback[-1][:3], + (__file__, + lineno, + 'test_handle_source_traceback')) + + # call_soon + h = loop.call_soon(noop) + check_source_traceback(h) + + # call_soon_threadsafe + h = loop.call_soon_threadsafe(noop) + check_source_traceback(h) + + # call_later + h = loop.call_later(0, noop) + check_source_traceback(h) + + # call_at + h = loop.call_later(0, noop) + check_source_traceback(h) + + @unittest.skipUnless(hasattr(collections.abc, 'Coroutine'), + 'No collections.abc.Coroutine') + def test_coroutine_like_object_debug_formatting(self): + # Test that asyncio can format coroutines that are instances of + # collections.abc.Coroutine, but lack cr_core or gi_code attributes + # (such as ones compiled with Cython). + + class Coro: + __name__ = 'AAA' + + def send(self, v): + pass + + def throw(self, *exc): + pass + + def close(self): + pass + + def __await__(self): + pass + + coro = Coro() + self.assertTrue(asyncio.iscoroutine(coro)) + self.assertEqual(coroutines._format_coroutine(coro), 'AAA()') + + coro.__qualname__ = 'BBB' + self.assertEqual(coroutines._format_coroutine(coro), 'BBB()') + + coro.cr_running = True + self.assertEqual(coroutines._format_coroutine(coro), 'BBB() running') + + +class TimerTests(unittest.TestCase): + + def setUp(self): + super().setUp() + self.loop = mock.Mock() + + def test_hash(self): + when = time.monotonic() + h = asyncio.TimerHandle(when, lambda: False, (), + mock.Mock()) + self.assertEqual(hash(h), hash(when)) + + def test_timer(self): + def callback(*args): + return args + + args = (1, 2, 3) + when = time.monotonic() + h = asyncio.TimerHandle(when, callback, args, mock.Mock()) + self.assertIs(h._callback, callback) + self.assertIs(h._args, args) + self.assertFalse(h._cancelled) + + # cancel + h.cancel() + self.assertTrue(h._cancelled) + self.assertIsNone(h._callback) + self.assertIsNone(h._args) + + # when cannot be None + self.assertRaises(AssertionError, + asyncio.TimerHandle, None, callback, args, + self.loop) + + def test_timer_repr(self): + self.loop.get_debug.return_value = False + + # simple function + h = asyncio.TimerHandle(123, noop, (), self.loop) + src = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' % src) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '') + + def test_timer_repr_debug(self): + self.loop.get_debug.return_value = True + + # simple function + create_filename = __file__ + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.TimerHandle(123, noop, (), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + + def test_timer_comparison(self): + def callback(*args): + return args + + when = time.monotonic() + + h1 = asyncio.TimerHandle(when, callback, (), self.loop) + h2 = asyncio.TimerHandle(when, callback, (), self.loop) + # TODO: Use assertLess etc. + self.assertFalse(h1 < h2) + self.assertFalse(h2 < h1) + self.assertTrue(h1 <= h2) + self.assertTrue(h2 <= h1) + self.assertFalse(h1 > h2) + self.assertFalse(h2 > h1) + self.assertTrue(h1 >= h2) + self.assertTrue(h2 >= h1) + self.assertTrue(h1 == h2) + self.assertFalse(h1 != h2) + + h2.cancel() + self.assertFalse(h1 == h2) + + h1 = asyncio.TimerHandle(when, callback, (), self.loop) + h2 = asyncio.TimerHandle(when + 10.0, callback, (), self.loop) + self.assertTrue(h1 < h2) + self.assertFalse(h2 < h1) + self.assertTrue(h1 <= h2) + self.assertFalse(h2 <= h1) + self.assertFalse(h1 > h2) + self.assertTrue(h2 > h1) + self.assertFalse(h1 >= h2) + self.assertTrue(h2 >= h1) + self.assertFalse(h1 == h2) + self.assertTrue(h1 != h2) + + h3 = asyncio.Handle(callback, (), self.loop) + self.assertIs(NotImplemented, h1.__eq__(h3)) + self.assertIs(NotImplemented, h1.__ne__(h3)) + + +class AbstractEventLoopTests(unittest.TestCase): + + def test_not_implemented(self): + f = mock.Mock() + loop = asyncio.AbstractEventLoop() + self.assertRaises( + NotImplementedError, loop.run_forever) + self.assertRaises( + NotImplementedError, loop.run_until_complete, None) + self.assertRaises( + NotImplementedError, loop.stop) + self.assertRaises( + NotImplementedError, loop.is_running) + self.assertRaises( + NotImplementedError, loop.is_closed) + self.assertRaises( + NotImplementedError, loop.close) + self.assertRaises( + NotImplementedError, loop.create_task, None) + self.assertRaises( + NotImplementedError, loop.call_later, None, None) + self.assertRaises( + NotImplementedError, loop.call_at, f, f) + self.assertRaises( + NotImplementedError, loop.call_soon, None) + self.assertRaises( + NotImplementedError, loop.time) + self.assertRaises( + NotImplementedError, loop.call_soon_threadsafe, None) + self.assertRaises( + NotImplementedError, loop.run_in_executor, f, f) + self.assertRaises( + NotImplementedError, loop.set_default_executor, f) + self.assertRaises( + NotImplementedError, loop.getaddrinfo, 'localhost', 8080) + self.assertRaises( + NotImplementedError, loop.getnameinfo, ('localhost', 8080)) + self.assertRaises( + NotImplementedError, loop.create_connection, f) + self.assertRaises( + NotImplementedError, loop.create_server, f) + self.assertRaises( + NotImplementedError, loop.create_datagram_endpoint, f) + self.assertRaises( + NotImplementedError, loop.add_reader, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_reader, 1) + self.assertRaises( + NotImplementedError, loop.add_writer, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_writer, 1) + self.assertRaises( + NotImplementedError, loop.sock_recv, f, 10) + self.assertRaises( + NotImplementedError, loop.sock_sendall, f, 10) + self.assertRaises( + NotImplementedError, loop.sock_connect, f, f) + self.assertRaises( + NotImplementedError, loop.sock_accept, f) + self.assertRaises( + NotImplementedError, loop.add_signal_handler, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_signal_handler, 1) + self.assertRaises( + NotImplementedError, loop.remove_signal_handler, 1) + self.assertRaises( + NotImplementedError, loop.connect_read_pipe, f, + mock.sentinel.pipe) + self.assertRaises( + NotImplementedError, loop.connect_write_pipe, f, + mock.sentinel.pipe) + self.assertRaises( + NotImplementedError, loop.subprocess_shell, f, + mock.sentinel) + self.assertRaises( + NotImplementedError, loop.subprocess_exec, f) + self.assertRaises( + NotImplementedError, loop.set_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.default_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.call_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.get_debug) + self.assertRaises( + NotImplementedError, loop.set_debug, f) + + +class ProtocolsAbsTests(unittest.TestCase): + + def test_empty(self): + f = mock.Mock() + p = asyncio.Protocol() + self.assertIsNone(p.connection_made(f)) + self.assertIsNone(p.connection_lost(f)) + self.assertIsNone(p.data_received(f)) + self.assertIsNone(p.eof_received()) + + dp = asyncio.DatagramProtocol() + self.assertIsNone(dp.connection_made(f)) + self.assertIsNone(dp.connection_lost(f)) + self.assertIsNone(dp.error_received(f)) + self.assertIsNone(dp.datagram_received(f, f)) + + sp = asyncio.SubprocessProtocol() + self.assertIsNone(sp.connection_made(f)) + self.assertIsNone(sp.connection_lost(f)) + self.assertIsNone(sp.pipe_data_received(1, f)) + self.assertIsNone(sp.pipe_connection_lost(1, f)) + self.assertIsNone(sp.process_exited()) + + +class PolicyTests(unittest.TestCase): + + def test_event_loop_policy(self): + policy = asyncio.AbstractEventLoopPolicy() + self.assertRaises(NotImplementedError, policy.get_event_loop) + self.assertRaises(NotImplementedError, policy.set_event_loop, object()) + self.assertRaises(NotImplementedError, policy.new_event_loop) + self.assertRaises(NotImplementedError, policy.get_child_watcher) + self.assertRaises(NotImplementedError, policy.set_child_watcher, + object()) + + def test_get_event_loop(self): + policy = asyncio.DefaultEventLoopPolicy() + self.assertIsNone(policy._local._loop) + + loop = policy.get_event_loop() + self.assertIsInstance(loop, asyncio.AbstractEventLoop) + + self.assertIs(policy._local._loop, loop) + self.assertIs(loop, policy.get_event_loop()) + loop.close() + + def test_get_event_loop_calls_set_event_loop(self): + policy = asyncio.DefaultEventLoopPolicy() + + with mock.patch.object( + policy, "set_event_loop", + wraps=policy.set_event_loop) as m_set_event_loop: + + loop = policy.get_event_loop() + + # policy._local._loop must be set through .set_event_loop() + # (the unix DefaultEventLoopPolicy needs this call to attach + # the child watcher correctly) + m_set_event_loop.assert_called_with(loop) + + loop.close() + + def test_get_event_loop_after_set_none(self): + policy = asyncio.DefaultEventLoopPolicy() + policy.set_event_loop(None) + self.assertRaises(RuntimeError, policy.get_event_loop) + + @mock.patch('asyncio.events.threading.current_thread') + def test_get_event_loop_thread(self, m_current_thread): + + def f(): + policy = asyncio.DefaultEventLoopPolicy() + self.assertRaises(RuntimeError, policy.get_event_loop) + + th = threading.Thread(target=f) + th.start() + th.join() + + def test_new_event_loop(self): + policy = asyncio.DefaultEventLoopPolicy() + + loop = policy.new_event_loop() + self.assertIsInstance(loop, asyncio.AbstractEventLoop) + loop.close() + + def test_set_event_loop(self): + policy = asyncio.DefaultEventLoopPolicy() + old_loop = policy.get_event_loop() + + self.assertRaises(AssertionError, policy.set_event_loop, object()) + + loop = policy.new_event_loop() + policy.set_event_loop(loop) + self.assertIs(loop, policy.get_event_loop()) + self.assertIsNot(old_loop, policy.get_event_loop()) + loop.close() + old_loop.close() + + def test_get_event_loop_policy(self): + policy = asyncio.get_event_loop_policy() + self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + self.assertIs(policy, asyncio.get_event_loop_policy()) + + def test_set_event_loop_policy(self): + self.assertRaises( + AssertionError, asyncio.set_event_loop_policy, object()) + + old_policy = asyncio.get_event_loop_policy() + + policy = asyncio.DefaultEventLoopPolicy() + asyncio.set_event_loop_policy(policy) + self.assertIs(policy, asyncio.get_event_loop_policy()) + self.assertIsNot(policy, old_policy) + + def test_get_event_loop_returns_running_loop(self): + class Policy(asyncio.DefaultEventLoopPolicy): + def get_event_loop(self): + raise NotImplementedError + + loop = None + + old_policy = asyncio.get_event_loop_policy() + try: + asyncio.set_event_loop_policy(Policy()) + loop = asyncio.new_event_loop() + + async def func(): + self.assertIs(asyncio.get_event_loop(), loop) + + loop.run_until_complete(func()) + finally: + asyncio.set_event_loop_policy(old_policy) + if loop is not None: + loop.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_futures.py b/thirdparty/asyncio/tests/test_futures.py new file mode 100644 index 0000000..153b8ed --- /dev/null +++ b/thirdparty/asyncio/tests/test_futures.py @@ -0,0 +1,551 @@ +"""Tests for futures.py.""" + +import concurrent.futures +import re +import sys +import threading +import unittest +from unittest import mock + +import asyncio +from asyncio import test_utils +try: + from test import support +except ImportError: + from asyncio import test_support as support + + +def _fakefunc(f): + return f + +def first_cb(): + pass + +def last_cb(): + pass + + +class DuckFuture: + # Class that does not inherit from Future but aims to be duck-type + # compatible with it. + + _asyncio_future_blocking = False + __cancelled = False + __result = None + __exception = None + + def cancel(self): + if self.done(): + return False + self.__cancelled = True + return True + + def cancelled(self): + return self.__cancelled + + def done(self): + return (self.__cancelled + or self.__result is not None + or self.__exception is not None) + + def result(self): + assert not self.cancelled() + if self.__exception is not None: + raise self.__exception + return self.__result + + def exception(self): + assert not self.cancelled() + return self.__exception + + def set_result(self, result): + assert not self.done() + assert result is not None + self.__result = result + + def set_exception(self, exception): + assert not self.done() + assert exception is not None + self.__exception = exception + + def __iter__(self): + if not self.done(): + self._asyncio_future_blocking = True + yield self + assert self.done() + return self.result() + + +class DuckTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_wrap_future(self): + f = DuckFuture() + g = asyncio.wrap_future(f) + assert g is f + + def test_ensure_future(self): + f = DuckFuture() + g = asyncio.ensure_future(f) + assert g is f + + +class FutureTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_initial_state(self): + f = asyncio.Future(loop=self.loop) + self.assertFalse(f.cancelled()) + self.assertFalse(f.done()) + f.cancel() + self.assertTrue(f.cancelled()) + + def test_init_constructor_default_loop(self): + asyncio.set_event_loop(self.loop) + f = asyncio.Future() + self.assertIs(f._loop, self.loop) + + def test_constructor_positional(self): + # Make sure Future doesn't accept a positional argument + self.assertRaises(TypeError, asyncio.Future, 42) + + def test_cancel(self): + f = asyncio.Future(loop=self.loop) + self.assertTrue(f.cancel()) + self.assertTrue(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(asyncio.CancelledError, f.result) + self.assertRaises(asyncio.CancelledError, f.exception) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_result(self): + f = asyncio.Future(loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, f.result) + + f.set_result(42) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertEqual(f.result(), 42) + self.assertEqual(f.exception(), None) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_exception(self): + exc = RuntimeError() + f = asyncio.Future(loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, f.exception) + + # StopIteration cannot be raised into a Future - CPython issue26221 + self.assertRaisesRegex(TypeError, "StopIteration .* cannot be raised", + f.set_exception, StopIteration) + + f.set_exception(exc) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(RuntimeError, f.result) + self.assertEqual(f.exception(), exc) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_exception_class(self): + f = asyncio.Future(loop=self.loop) + f.set_exception(RuntimeError) + self.assertIsInstance(f.exception(), RuntimeError) + + def test_yield_from_twice(self): + f = asyncio.Future(loop=self.loop) + + def fixture(): + yield 'A' + x = yield from f + yield 'B', x + y = yield from f + yield 'C', y + + g = fixture() + self.assertEqual(next(g), 'A') # yield 'A'. + self.assertEqual(next(g), f) # First yield from f. + f.set_result(42) + self.assertEqual(next(g), ('B', 42)) # yield 'B', x. + # The second "yield from f" does not yield f. + self.assertEqual(next(g), ('C', 42)) # yield 'C', y. + + def test_future_repr(self): + self.loop.set_debug(True) + f_pending_debug = asyncio.Future(loop=self.loop) + frame = f_pending_debug._source_traceback[-1] + self.assertEqual(repr(f_pending_debug), + '' + % (frame[0], frame[1])) + f_pending_debug.cancel() + + self.loop.set_debug(False) + f_pending = asyncio.Future(loop=self.loop) + self.assertEqual(repr(f_pending), '') + f_pending.cancel() + + f_cancelled = asyncio.Future(loop=self.loop) + f_cancelled.cancel() + self.assertEqual(repr(f_cancelled), '') + + f_result = asyncio.Future(loop=self.loop) + f_result.set_result(4) + self.assertEqual(repr(f_result), '') + self.assertEqual(f_result.result(), 4) + + exc = RuntimeError() + f_exception = asyncio.Future(loop=self.loop) + f_exception.set_exception(exc) + self.assertEqual(repr(f_exception), + '') + self.assertIs(f_exception.exception(), exc) + + def func_repr(func): + filename, lineno = test_utils.get_function_source(func) + text = '%s() at %s:%s' % (func.__qualname__, filename, lineno) + return re.escape(text) + + f_one_callbacks = asyncio.Future(loop=self.loop) + f_one_callbacks.add_done_callback(_fakefunc) + fake_repr = func_repr(_fakefunc) + self.assertRegex(repr(f_one_callbacks), + r'' % fake_repr) + f_one_callbacks.cancel() + self.assertEqual(repr(f_one_callbacks), + '') + + f_two_callbacks = asyncio.Future(loop=self.loop) + f_two_callbacks.add_done_callback(first_cb) + f_two_callbacks.add_done_callback(last_cb) + first_repr = func_repr(first_cb) + last_repr = func_repr(last_cb) + self.assertRegex(repr(f_two_callbacks), + r'' + % (first_repr, last_repr)) + + f_many_callbacks = asyncio.Future(loop=self.loop) + f_many_callbacks.add_done_callback(first_cb) + for i in range(8): + f_many_callbacks.add_done_callback(_fakefunc) + f_many_callbacks.add_done_callback(last_cb) + cb_regex = r'%s, <8 more>, %s' % (first_repr, last_repr) + self.assertRegex(repr(f_many_callbacks), + r'' % cb_regex) + f_many_callbacks.cancel() + self.assertEqual(repr(f_many_callbacks), + '') + + def test_copy_state(self): + from asyncio.futures import _copy_future_state + + f = asyncio.Future(loop=self.loop) + f.set_result(10) + + newf = asyncio.Future(loop=self.loop) + _copy_future_state(f, newf) + self.assertTrue(newf.done()) + self.assertEqual(newf.result(), 10) + + f_exception = asyncio.Future(loop=self.loop) + f_exception.set_exception(RuntimeError()) + + newf_exception = asyncio.Future(loop=self.loop) + _copy_future_state(f_exception, newf_exception) + self.assertTrue(newf_exception.done()) + self.assertRaises(RuntimeError, newf_exception.result) + + f_cancelled = asyncio.Future(loop=self.loop) + f_cancelled.cancel() + + newf_cancelled = asyncio.Future(loop=self.loop) + _copy_future_state(f_cancelled, newf_cancelled) + self.assertTrue(newf_cancelled.cancelled()) + + def test_iter(self): + fut = asyncio.Future(loop=self.loop) + + def coro(): + yield from fut + + def test(): + arg1, arg2 = coro() + + self.assertRaises(AssertionError, test) + fut.cancel() + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_abandoned(self, m_log): + fut = asyncio.Future(loop=self.loop) + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_result_unretrieved(self, m_log): + fut = asyncio.Future(loop=self.loop) + fut.set_result(42) + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_result_retrieved(self, m_log): + fut = asyncio.Future(loop=self.loop) + fut.set_result(42) + fut.result() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_unretrieved(self, m_log): + fut = asyncio.Future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + del fut + test_utils.run_briefly(self.loop) + support.gc_collect() + self.assertTrue(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_retrieved(self, m_log): + fut = asyncio.Future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + fut.exception() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_result_retrieved(self, m_log): + fut = asyncio.Future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + self.assertRaises(RuntimeError, fut.result) + del fut + self.assertFalse(m_log.error.called) + + def test_wrap_future(self): + + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + f2 = asyncio.wrap_future(f1, loop=self.loop) + res, ident = self.loop.run_until_complete(f2) + self.assertIsInstance(f2, asyncio.Future) + self.assertEqual(res, 'oi') + self.assertNotEqual(ident, threading.get_ident()) + + def test_wrap_future_future(self): + f1 = asyncio.Future(loop=self.loop) + f2 = asyncio.wrap_future(f1) + self.assertIs(f1, f2) + + def test_wrap_future_use_global_loop(self): + with mock.patch('asyncio.futures.events') as events: + events.get_event_loop = lambda: self.loop + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + f2 = asyncio.wrap_future(f1) + self.assertIs(self.loop, f2._loop) + + def test_wrap_future_cancel(self): + f1 = concurrent.futures.Future() + f2 = asyncio.wrap_future(f1, loop=self.loop) + f2.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(f1.cancelled()) + self.assertTrue(f2.cancelled()) + + def test_wrap_future_cancel2(self): + f1 = concurrent.futures.Future() + f2 = asyncio.wrap_future(f1, loop=self.loop) + f1.set_result(42) + f2.cancel() + test_utils.run_briefly(self.loop) + self.assertFalse(f1.cancelled()) + self.assertEqual(f1.result(), 42) + self.assertTrue(f2.cancelled()) + + def test_future_source_traceback(self): + self.loop.set_debug(True) + + future = asyncio.Future(loop=self.loop) + lineno = sys._getframe().f_lineno - 1 + self.assertIsInstance(future._source_traceback, list) + self.assertEqual(future._source_traceback[-1][:3], + (__file__, + lineno, + 'test_future_source_traceback')) + + @mock.patch('asyncio.base_events.logger') + def check_future_exception_never_retrieved(self, debug, m_log): + self.loop.set_debug(debug) + + def memory_error(): + try: + raise MemoryError() + except BaseException as exc: + return exc + exc = memory_error() + + future = asyncio.Future(loop=self.loop) + if debug: + source_traceback = future._source_traceback + future.set_exception(exc) + future = None + test_utils.run_briefly(self.loop) + support.gc_collect() + + if sys.version_info >= (3, 4): + if debug: + frame = source_traceback[-1] + regex = (r'^Future exception was never retrieved\n' + r'future: \n' + r'source_traceback: Object ' + r'created at \(most recent call last\):\n' + r' File' + r'.*\n' + r' File "{filename}", line {lineno}, ' + r'in check_future_exception_never_retrieved\n' + r' future = asyncio\.Future\(loop=self\.loop\)$' + ).format(filename=re.escape(frame[0]), + lineno=frame[1]) + else: + regex = (r'^Future exception was never retrieved\n' + r'future: ' + r'$' + ) + exc_info = (type(exc), exc, exc.__traceback__) + m_log.error.assert_called_once_with(mock.ANY, exc_info=exc_info) + else: + if debug: + frame = source_traceback[-1] + regex = (r'^Future/Task exception was never retrieved\n' + r'Future/Task created at \(most recent call last\):\n' + r' File' + r'.*\n' + r' File "{filename}", line {lineno}, ' + r'in check_future_exception_never_retrieved\n' + r' future = asyncio\.Future\(loop=self\.loop\)\n' + r'Traceback \(most recent call last\):\n' + r'.*\n' + r'MemoryError$' + ).format(filename=re.escape(frame[0]), + lineno=frame[1]) + else: + regex = (r'^Future/Task exception was never retrieved\n' + r'Traceback \(most recent call last\):\n' + r'.*\n' + r'MemoryError$' + ) + m_log.error.assert_called_once_with(mock.ANY, exc_info=False) + message = m_log.error.call_args[0][0] + self.assertRegex(message, re.compile(regex, re.DOTALL)) + + def test_future_exception_never_retrieved(self): + self.check_future_exception_never_retrieved(False) + + def test_future_exception_never_retrieved_debug(self): + self.check_future_exception_never_retrieved(True) + + def test_set_result_unless_cancelled(self): + from asyncio import futures + fut = asyncio.Future(loop=self.loop) + fut.cancel() + futures._set_result_unless_cancelled(fut, 2) + self.assertTrue(fut.cancelled()) + + +class FutureDoneCallbackTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + def run_briefly(self): + test_utils.run_briefly(self.loop) + + def _make_callback(self, bag, thing): + # Create a callback function that appends thing to bag. + def bag_appender(future): + bag.append(thing) + return bag_appender + + def _new_future(self): + return asyncio.Future(loop=self.loop) + + def test_callbacks_invoked_on_set_result(self): + bag = [] + f = self._new_future() + f.add_done_callback(self._make_callback(bag, 42)) + f.add_done_callback(self._make_callback(bag, 17)) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [42, 17]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_invoked_on_set_exception(self): + bag = [] + f = self._new_future() + f.add_done_callback(self._make_callback(bag, 100)) + + self.assertEqual(bag, []) + exc = RuntimeError() + f.set_exception(exc) + + self.run_briefly() + + self.assertEqual(bag, [100]) + self.assertEqual(f.exception(), exc) + + def test_remove_done_callback(self): + bag = [] + f = self._new_future() + cb1 = self._make_callback(bag, 1) + cb2 = self._make_callback(bag, 2) + cb3 = self._make_callback(bag, 3) + + # Add one cb1 and one cb2. + f.add_done_callback(cb1) + f.add_done_callback(cb2) + + # One instance of cb2 removed. Now there's only one cb1. + self.assertEqual(f.remove_done_callback(cb2), 1) + + # Never had any cb3 in there. + self.assertEqual(f.remove_done_callback(cb3), 0) + + # After this there will be 6 instances of cb1 and one of cb2. + f.add_done_callback(cb2) + for i in range(5): + f.add_done_callback(cb1) + + # Remove all instances of cb1. One cb2 remains. + self.assertEqual(f.remove_done_callback(cb1), 6) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [2]) + self.assertEqual(f.result(), 'foo') + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_locks.py b/thirdparty/asyncio/tests/test_locks.py new file mode 100644 index 0000000..152948c --- /dev/null +++ b/thirdparty/asyncio/tests/test_locks.py @@ -0,0 +1,921 @@ +"""Tests for lock.py""" + +import unittest +from unittest import mock +import re + +import asyncio +from asyncio import test_utils + +STR_RGX_REPR = ( + r'^<(?P.*?) object at (?P
.*?)' + r'\[(?P' + r'(set|unset|locked|unlocked)(,value:\d)?(,waiters:\d+)?' + r')\]>\Z' +) +RGX_REPR = re.compile(STR_RGX_REPR) + + +class LockTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + def test_ctor_loop(self): + loop = mock.Mock() + lock = asyncio.Lock(loop=loop) + self.assertIs(lock._loop, loop) + + lock = asyncio.Lock(loop=self.loop) + self.assertIs(lock._loop, self.loop) + + def test_ctor_noloop(self): + asyncio.set_event_loop(self.loop) + lock = asyncio.Lock() + self.assertIs(lock._loop, self.loop) + + def test_repr(self): + lock = asyncio.Lock(loop=self.loop) + self.assertTrue(repr(lock).endswith('[unlocked]>')) + self.assertTrue(RGX_REPR.match(repr(lock))) + + @asyncio.coroutine + def acquire_lock(): + yield from lock + + self.loop.run_until_complete(acquire_lock()) + self.assertTrue(repr(lock).endswith('[locked]>')) + self.assertTrue(RGX_REPR.match(repr(lock))) + + def test_lock(self): + lock = asyncio.Lock(loop=self.loop) + + @asyncio.coroutine + def acquire_lock(): + return (yield from lock) + + res = self.loop.run_until_complete(acquire_lock()) + + self.assertTrue(res) + self.assertTrue(lock.locked()) + + lock.release() + self.assertFalse(lock.locked()) + + def test_acquire(self): + lock = asyncio.Lock(loop=self.loop) + result = [] + + self.assertTrue(self.loop.run_until_complete(lock.acquire())) + + @asyncio.coroutine + def c1(result): + if (yield from lock.acquire()): + result.append(1) + return True + + @asyncio.coroutine + def c2(result): + if (yield from lock.acquire()): + result.append(2) + return True + + @asyncio.coroutine + def c3(result): + if (yield from lock.acquire()): + result.append(3) + return True + + t1 = asyncio.Task(c1(result), loop=self.loop) + t2 = asyncio.Task(c2(result), loop=self.loop) + + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + + lock.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1], result) + + test_utils.run_briefly(self.loop) + self.assertEqual([1], result) + + t3 = asyncio.Task(c3(result), loop=self.loop) + + lock.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1, 2], result) + + lock.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1, 2, 3], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + def test_acquire_cancel(self): + lock = asyncio.Lock(loop=self.loop) + self.assertTrue(self.loop.run_until_complete(lock.acquire())) + + task = asyncio.Task(lock.acquire(), loop=self.loop) + self.loop.call_soon(task.cancel) + self.assertRaises( + asyncio.CancelledError, + self.loop.run_until_complete, task) + self.assertFalse(lock._waiters) + + def test_cancel_race(self): + # Several tasks: + # - A acquires the lock + # - B is blocked in acquire() + # - C is blocked in acquire() + # + # Now, concurrently: + # - B is cancelled + # - A releases the lock + # + # If B's waiter is marked cancelled but not yet removed from + # _waiters, A's release() call will crash when trying to set + # B's waiter; instead, it should move on to C's waiter. + + # Setup: A has the lock, b and c are waiting. + lock = asyncio.Lock(loop=self.loop) + + @asyncio.coroutine + def lockit(name, blocker): + yield from lock.acquire() + try: + if blocker is not None: + yield from blocker + finally: + lock.release() + + fa = asyncio.Future(loop=self.loop) + ta = asyncio.Task(lockit('A', fa), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertTrue(lock.locked()) + tb = asyncio.Task(lockit('B', None), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertEqual(len(lock._waiters), 1) + tc = asyncio.Task(lockit('C', None), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertEqual(len(lock._waiters), 2) + + # Create the race and check. + # Without the fix this failed at the last assert. + fa.set_result(None) + tb.cancel() + self.assertTrue(lock._waiters[0].cancelled()) + test_utils.run_briefly(self.loop) + self.assertFalse(lock.locked()) + self.assertTrue(ta.done()) + self.assertTrue(tb.cancelled()) + self.assertTrue(tc.done()) + + def test_release_not_acquired(self): + lock = asyncio.Lock(loop=self.loop) + + self.assertRaises(RuntimeError, lock.release) + + def test_release_no_waiters(self): + lock = asyncio.Lock(loop=self.loop) + self.loop.run_until_complete(lock.acquire()) + self.assertTrue(lock.locked()) + + lock.release() + self.assertFalse(lock.locked()) + + def test_context_manager(self): + lock = asyncio.Lock(loop=self.loop) + + @asyncio.coroutine + def acquire_lock(): + return (yield from lock) + + with self.loop.run_until_complete(acquire_lock()): + self.assertTrue(lock.locked()) + + self.assertFalse(lock.locked()) + + def test_context_manager_cant_reuse(self): + lock = asyncio.Lock(loop=self.loop) + + @asyncio.coroutine + def acquire_lock(): + return (yield from lock) + + # This spells "yield from lock" outside a generator. + cm = self.loop.run_until_complete(acquire_lock()) + with cm: + self.assertTrue(lock.locked()) + + self.assertFalse(lock.locked()) + + with self.assertRaises(AttributeError): + with cm: + pass + + def test_context_manager_no_yield(self): + lock = asyncio.Lock(loop=self.loop) + + try: + with lock: + self.fail('RuntimeError is not raised in with expression') + except RuntimeError as err: + self.assertEqual( + str(err), + '"yield from" should be used as context manager expression') + + self.assertFalse(lock.locked()) + + +class EventTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + def test_ctor_loop(self): + loop = mock.Mock() + ev = asyncio.Event(loop=loop) + self.assertIs(ev._loop, loop) + + ev = asyncio.Event(loop=self.loop) + self.assertIs(ev._loop, self.loop) + + def test_ctor_noloop(self): + asyncio.set_event_loop(self.loop) + ev = asyncio.Event() + self.assertIs(ev._loop, self.loop) + + def test_repr(self): + ev = asyncio.Event(loop=self.loop) + self.assertTrue(repr(ev).endswith('[unset]>')) + match = RGX_REPR.match(repr(ev)) + self.assertEqual(match.group('extras'), 'unset') + + ev.set() + self.assertTrue(repr(ev).endswith('[set]>')) + self.assertTrue(RGX_REPR.match(repr(ev))) + + ev._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(ev)) + self.assertTrue(RGX_REPR.match(repr(ev))) + + def test_wait(self): + ev = asyncio.Event(loop=self.loop) + self.assertFalse(ev.is_set()) + + result = [] + + @asyncio.coroutine + def c1(result): + if (yield from ev.wait()): + result.append(1) + + @asyncio.coroutine + def c2(result): + if (yield from ev.wait()): + result.append(2) + + @asyncio.coroutine + def c3(result): + if (yield from ev.wait()): + result.append(3) + + t1 = asyncio.Task(c1(result), loop=self.loop) + t2 = asyncio.Task(c2(result), loop=self.loop) + + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + + t3 = asyncio.Task(c3(result), loop=self.loop) + + ev.set() + test_utils.run_briefly(self.loop) + self.assertEqual([3, 1, 2], result) + + self.assertTrue(t1.done()) + self.assertIsNone(t1.result()) + self.assertTrue(t2.done()) + self.assertIsNone(t2.result()) + self.assertTrue(t3.done()) + self.assertIsNone(t3.result()) + + def test_wait_on_set(self): + ev = asyncio.Event(loop=self.loop) + ev.set() + + res = self.loop.run_until_complete(ev.wait()) + self.assertTrue(res) + + def test_wait_cancel(self): + ev = asyncio.Event(loop=self.loop) + + wait = asyncio.Task(ev.wait(), loop=self.loop) + self.loop.call_soon(wait.cancel) + self.assertRaises( + asyncio.CancelledError, + self.loop.run_until_complete, wait) + self.assertFalse(ev._waiters) + + def test_clear(self): + ev = asyncio.Event(loop=self.loop) + self.assertFalse(ev.is_set()) + + ev.set() + self.assertTrue(ev.is_set()) + + ev.clear() + self.assertFalse(ev.is_set()) + + def test_clear_with_waiters(self): + ev = asyncio.Event(loop=self.loop) + result = [] + + @asyncio.coroutine + def c1(result): + if (yield from ev.wait()): + result.append(1) + return True + + t = asyncio.Task(c1(result), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + + ev.set() + ev.clear() + self.assertFalse(ev.is_set()) + + ev.set() + ev.set() + self.assertEqual(1, len(ev._waiters)) + + test_utils.run_briefly(self.loop) + self.assertEqual([1], result) + self.assertEqual(0, len(ev._waiters)) + + self.assertTrue(t.done()) + self.assertTrue(t.result()) + + +class ConditionTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + def test_ctor_loop(self): + loop = mock.Mock() + cond = asyncio.Condition(loop=loop) + self.assertIs(cond._loop, loop) + + cond = asyncio.Condition(loop=self.loop) + self.assertIs(cond._loop, self.loop) + + def test_ctor_noloop(self): + asyncio.set_event_loop(self.loop) + cond = asyncio.Condition() + self.assertIs(cond._loop, self.loop) + + def test_wait(self): + cond = asyncio.Condition(loop=self.loop) + result = [] + + @asyncio.coroutine + def c1(result): + yield from cond.acquire() + if (yield from cond.wait()): + result.append(1) + return True + + @asyncio.coroutine + def c2(result): + yield from cond.acquire() + if (yield from cond.wait()): + result.append(2) + return True + + @asyncio.coroutine + def c3(result): + yield from cond.acquire() + if (yield from cond.wait()): + result.append(3) + return True + + t1 = asyncio.Task(c1(result), loop=self.loop) + t2 = asyncio.Task(c2(result), loop=self.loop) + t3 = asyncio.Task(c3(result), loop=self.loop) + + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + self.assertFalse(cond.locked()) + + self.assertTrue(self.loop.run_until_complete(cond.acquire())) + cond.notify() + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + self.assertTrue(cond.locked()) + + cond.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1], result) + self.assertTrue(cond.locked()) + + cond.notify(2) + test_utils.run_briefly(self.loop) + self.assertEqual([1], result) + self.assertTrue(cond.locked()) + + cond.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1, 2], result) + self.assertTrue(cond.locked()) + + cond.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1, 2, 3], result) + self.assertTrue(cond.locked()) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + def test_wait_cancel(self): + cond = asyncio.Condition(loop=self.loop) + self.loop.run_until_complete(cond.acquire()) + + wait = asyncio.Task(cond.wait(), loop=self.loop) + self.loop.call_soon(wait.cancel) + self.assertRaises( + asyncio.CancelledError, + self.loop.run_until_complete, wait) + self.assertFalse(cond._waiters) + self.assertTrue(cond.locked()) + + def test_wait_cancel_contested(self): + cond = asyncio.Condition(loop=self.loop) + + self.loop.run_until_complete(cond.acquire()) + self.assertTrue(cond.locked()) + + wait_task = asyncio.Task(cond.wait(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertFalse(cond.locked()) + + # Notify, but contest the lock before cancelling + self.loop.run_until_complete(cond.acquire()) + self.assertTrue(cond.locked()) + cond.notify() + self.loop.call_soon(wait_task.cancel) + self.loop.call_soon(cond.release) + + try: + self.loop.run_until_complete(wait_task) + except asyncio.CancelledError: + # Should not happen, since no cancellation points + pass + + self.assertTrue(cond.locked()) + + def test_wait_unacquired(self): + cond = asyncio.Condition(loop=self.loop) + self.assertRaises( + RuntimeError, + self.loop.run_until_complete, cond.wait()) + + def test_wait_for(self): + cond = asyncio.Condition(loop=self.loop) + presult = False + + def predicate(): + return presult + + result = [] + + @asyncio.coroutine + def c1(result): + yield from cond.acquire() + if (yield from cond.wait_for(predicate)): + result.append(1) + cond.release() + return True + + t = asyncio.Task(c1(result), loop=self.loop) + + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + + self.loop.run_until_complete(cond.acquire()) + cond.notify() + cond.release() + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + + presult = True + self.loop.run_until_complete(cond.acquire()) + cond.notify() + cond.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1], result) + + self.assertTrue(t.done()) + self.assertTrue(t.result()) + + def test_wait_for_unacquired(self): + cond = asyncio.Condition(loop=self.loop) + + # predicate can return true immediately + res = self.loop.run_until_complete(cond.wait_for(lambda: [1, 2, 3])) + self.assertEqual([1, 2, 3], res) + + self.assertRaises( + RuntimeError, + self.loop.run_until_complete, + cond.wait_for(lambda: False)) + + def test_notify(self): + cond = asyncio.Condition(loop=self.loop) + result = [] + + @asyncio.coroutine + def c1(result): + yield from cond.acquire() + if (yield from cond.wait()): + result.append(1) + cond.release() + return True + + @asyncio.coroutine + def c2(result): + yield from cond.acquire() + if (yield from cond.wait()): + result.append(2) + cond.release() + return True + + @asyncio.coroutine + def c3(result): + yield from cond.acquire() + if (yield from cond.wait()): + result.append(3) + cond.release() + return True + + t1 = asyncio.Task(c1(result), loop=self.loop) + t2 = asyncio.Task(c2(result), loop=self.loop) + t3 = asyncio.Task(c3(result), loop=self.loop) + + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + + self.loop.run_until_complete(cond.acquire()) + cond.notify(1) + cond.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1], result) + + self.loop.run_until_complete(cond.acquire()) + cond.notify(1) + cond.notify(2048) + cond.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1, 2, 3], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + def test_notify_all(self): + cond = asyncio.Condition(loop=self.loop) + + result = [] + + @asyncio.coroutine + def c1(result): + yield from cond.acquire() + if (yield from cond.wait()): + result.append(1) + cond.release() + return True + + @asyncio.coroutine + def c2(result): + yield from cond.acquire() + if (yield from cond.wait()): + result.append(2) + cond.release() + return True + + t1 = asyncio.Task(c1(result), loop=self.loop) + t2 = asyncio.Task(c2(result), loop=self.loop) + + test_utils.run_briefly(self.loop) + self.assertEqual([], result) + + self.loop.run_until_complete(cond.acquire()) + cond.notify_all() + cond.release() + test_utils.run_briefly(self.loop) + self.assertEqual([1, 2], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + + def test_notify_unacquired(self): + cond = asyncio.Condition(loop=self.loop) + self.assertRaises(RuntimeError, cond.notify) + + def test_notify_all_unacquired(self): + cond = asyncio.Condition(loop=self.loop) + self.assertRaises(RuntimeError, cond.notify_all) + + def test_repr(self): + cond = asyncio.Condition(loop=self.loop) + self.assertTrue('unlocked' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + self.loop.run_until_complete(cond.acquire()) + self.assertTrue('locked' in repr(cond)) + + cond._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + cond._waiters.append(mock.Mock()) + self.assertTrue('waiters:2' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + def test_context_manager(self): + cond = asyncio.Condition(loop=self.loop) + + @asyncio.coroutine + def acquire_cond(): + return (yield from cond) + + with self.loop.run_until_complete(acquire_cond()): + self.assertTrue(cond.locked()) + + self.assertFalse(cond.locked()) + + def test_context_manager_no_yield(self): + cond = asyncio.Condition(loop=self.loop) + + try: + with cond: + self.fail('RuntimeError is not raised in with expression') + except RuntimeError as err: + self.assertEqual( + str(err), + '"yield from" should be used as context manager expression') + + self.assertFalse(cond.locked()) + + def test_explicit_lock(self): + lock = asyncio.Lock(loop=self.loop) + cond = asyncio.Condition(lock, loop=self.loop) + + self.assertIs(cond._lock, lock) + self.assertIs(cond._loop, lock._loop) + + def test_ambiguous_loops(self): + loop = self.new_test_loop() + self.addCleanup(loop.close) + + lock = asyncio.Lock(loop=self.loop) + with self.assertRaises(ValueError): + asyncio.Condition(lock, loop=loop) + + +class SemaphoreTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + def test_ctor_loop(self): + loop = mock.Mock() + sem = asyncio.Semaphore(loop=loop) + self.assertIs(sem._loop, loop) + + sem = asyncio.Semaphore(loop=self.loop) + self.assertIs(sem._loop, self.loop) + + def test_ctor_noloop(self): + asyncio.set_event_loop(self.loop) + sem = asyncio.Semaphore() + self.assertIs(sem._loop, self.loop) + + def test_initial_value_zero(self): + sem = asyncio.Semaphore(0, loop=self.loop) + self.assertTrue(sem.locked()) + + def test_repr(self): + sem = asyncio.Semaphore(loop=self.loop) + self.assertTrue(repr(sem).endswith('[unlocked,value:1]>')) + self.assertTrue(RGX_REPR.match(repr(sem))) + + self.loop.run_until_complete(sem.acquire()) + self.assertTrue(repr(sem).endswith('[locked]>')) + self.assertTrue('waiters' not in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + sem._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + sem._waiters.append(mock.Mock()) + self.assertTrue('waiters:2' in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + def test_semaphore(self): + sem = asyncio.Semaphore(loop=self.loop) + self.assertEqual(1, sem._value) + + @asyncio.coroutine + def acquire_lock(): + return (yield from sem) + + res = self.loop.run_until_complete(acquire_lock()) + + self.assertTrue(res) + self.assertTrue(sem.locked()) + self.assertEqual(0, sem._value) + + sem.release() + self.assertFalse(sem.locked()) + self.assertEqual(1, sem._value) + + def test_semaphore_value(self): + self.assertRaises(ValueError, asyncio.Semaphore, -1) + + def test_acquire(self): + sem = asyncio.Semaphore(3, loop=self.loop) + result = [] + + self.assertTrue(self.loop.run_until_complete(sem.acquire())) + self.assertTrue(self.loop.run_until_complete(sem.acquire())) + self.assertFalse(sem.locked()) + + @asyncio.coroutine + def c1(result): + yield from sem.acquire() + result.append(1) + return True + + @asyncio.coroutine + def c2(result): + yield from sem.acquire() + result.append(2) + return True + + @asyncio.coroutine + def c3(result): + yield from sem.acquire() + result.append(3) + return True + + @asyncio.coroutine + def c4(result): + yield from sem.acquire() + result.append(4) + return True + + t1 = asyncio.Task(c1(result), loop=self.loop) + t2 = asyncio.Task(c2(result), loop=self.loop) + t3 = asyncio.Task(c3(result), loop=self.loop) + + test_utils.run_briefly(self.loop) + self.assertEqual([1], result) + self.assertTrue(sem.locked()) + self.assertEqual(2, len(sem._waiters)) + self.assertEqual(0, sem._value) + + t4 = asyncio.Task(c4(result), loop=self.loop) + + sem.release() + sem.release() + self.assertEqual(2, sem._value) + + test_utils.run_briefly(self.loop) + self.assertEqual(0, sem._value) + self.assertEqual(3, len(result)) + self.assertTrue(sem.locked()) + self.assertEqual(1, len(sem._waiters)) + self.assertEqual(0, sem._value) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + race_tasks = [t2, t3, t4] + done_tasks = [t for t in race_tasks if t.done() and t.result()] + self.assertTrue(2, len(done_tasks)) + + # cleanup locked semaphore + sem.release() + self.loop.run_until_complete(asyncio.gather(*race_tasks)) + + def test_acquire_cancel(self): + sem = asyncio.Semaphore(loop=self.loop) + self.loop.run_until_complete(sem.acquire()) + + acquire = asyncio.Task(sem.acquire(), loop=self.loop) + self.loop.call_soon(acquire.cancel) + self.assertRaises( + asyncio.CancelledError, + self.loop.run_until_complete, acquire) + self.assertTrue((not sem._waiters) or + all(waiter.done() for waiter in sem._waiters)) + + def test_acquire_cancel_before_awoken(self): + sem = asyncio.Semaphore(value=0, loop=self.loop) + + t1 = asyncio.Task(sem.acquire(), loop=self.loop) + t2 = asyncio.Task(sem.acquire(), loop=self.loop) + t3 = asyncio.Task(sem.acquire(), loop=self.loop) + t4 = asyncio.Task(sem.acquire(), loop=self.loop) + + test_utils.run_briefly(self.loop) + + sem.release() + t1.cancel() + t2.cancel() + + test_utils.run_briefly(self.loop) + num_done = sum(t.done() for t in [t3, t4]) + self.assertEqual(num_done, 1) + + t3.cancel() + t4.cancel() + test_utils.run_briefly(self.loop) + + def test_acquire_hang(self): + sem = asyncio.Semaphore(value=0, loop=self.loop) + + t1 = asyncio.Task(sem.acquire(), loop=self.loop) + t2 = asyncio.Task(sem.acquire(), loop=self.loop) + + test_utils.run_briefly(self.loop) + + sem.release() + t1.cancel() + + test_utils.run_briefly(self.loop) + self.assertTrue(sem.locked()) + + def test_release_not_acquired(self): + sem = asyncio.BoundedSemaphore(loop=self.loop) + + self.assertRaises(ValueError, sem.release) + + def test_release_no_waiters(self): + sem = asyncio.Semaphore(loop=self.loop) + self.loop.run_until_complete(sem.acquire()) + self.assertTrue(sem.locked()) + + sem.release() + self.assertFalse(sem.locked()) + + def test_context_manager(self): + sem = asyncio.Semaphore(2, loop=self.loop) + + @asyncio.coroutine + def acquire_lock(): + return (yield from sem) + + with self.loop.run_until_complete(acquire_lock()): + self.assertFalse(sem.locked()) + self.assertEqual(1, sem._value) + + with self.loop.run_until_complete(acquire_lock()): + self.assertTrue(sem.locked()) + + self.assertEqual(2, sem._value) + + def test_context_manager_no_yield(self): + sem = asyncio.Semaphore(2, loop=self.loop) + + try: + with sem: + self.fail('RuntimeError is not raised in with expression') + except RuntimeError as err: + self.assertEqual( + str(err), + '"yield from" should be used as context manager expression') + + self.assertEqual(2, sem._value) + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_pep492.py b/thirdparty/asyncio/tests/test_pep492.py new file mode 100644 index 0000000..d5b8522 --- /dev/null +++ b/thirdparty/asyncio/tests/test_pep492.py @@ -0,0 +1,232 @@ +"""Tests support for new syntax introduced by PEP 492.""" + +import collections.abc +import types +import unittest + +try: + from test import support +except ImportError: + from asyncio import test_support as support +from unittest import mock + +import asyncio +from asyncio import test_utils + + +class BaseTest(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.BaseEventLoop() + self.loop._process_events = mock.Mock() + self.loop._selector = mock.Mock() + self.loop._selector.select.return_value = () + self.set_event_loop(self.loop) + + +class LockTests(BaseTest): + + def test_context_manager_async_with(self): + primitives = [ + asyncio.Lock(loop=self.loop), + asyncio.Condition(loop=self.loop), + asyncio.Semaphore(loop=self.loop), + asyncio.BoundedSemaphore(loop=self.loop), + ] + + async def test(lock): + await asyncio.sleep(0.01, loop=self.loop) + self.assertFalse(lock.locked()) + async with lock as _lock: + self.assertIs(_lock, None) + self.assertTrue(lock.locked()) + await asyncio.sleep(0.01, loop=self.loop) + self.assertTrue(lock.locked()) + self.assertFalse(lock.locked()) + + for primitive in primitives: + self.loop.run_until_complete(test(primitive)) + self.assertFalse(primitive.locked()) + + def test_context_manager_with_await(self): + primitives = [ + asyncio.Lock(loop=self.loop), + asyncio.Condition(loop=self.loop), + asyncio.Semaphore(loop=self.loop), + asyncio.BoundedSemaphore(loop=self.loop), + ] + + async def test(lock): + await asyncio.sleep(0.01, loop=self.loop) + self.assertFalse(lock.locked()) + with await lock as _lock: + self.assertIs(_lock, None) + self.assertTrue(lock.locked()) + await asyncio.sleep(0.01, loop=self.loop) + self.assertTrue(lock.locked()) + self.assertFalse(lock.locked()) + + for primitive in primitives: + self.loop.run_until_complete(test(primitive)) + self.assertFalse(primitive.locked()) + + +class StreamReaderTests(BaseTest): + + def test_readline(self): + DATA = b'line1\nline2\nline3' + + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(DATA) + stream.feed_eof() + + async def reader(): + data = [] + async for line in stream: + data.append(line) + return data + + data = self.loop.run_until_complete(reader()) + self.assertEqual(data, [b'line1\n', b'line2\n', b'line3']) + + +class CoroutineTests(BaseTest): + + def test_iscoroutine(self): + async def foo(): pass + + f = foo() + try: + self.assertTrue(asyncio.iscoroutine(f)) + finally: + f.close() # silence warning + + # Test that asyncio.iscoroutine() uses collections.abc.Coroutine + class FakeCoro: + def send(self, value): pass + def throw(self, typ, val=None, tb=None): pass + def close(self): pass + def __await__(self): yield + + self.assertTrue(asyncio.iscoroutine(FakeCoro())) + + def test_iscoroutinefunction(self): + async def foo(): pass + self.assertTrue(asyncio.iscoroutinefunction(foo)) + + def test_function_returning_awaitable(self): + class Awaitable: + def __await__(self): + return ('spam',) + + @asyncio.coroutine + def func(): + return Awaitable() + + coro = func() + self.assertEqual(coro.send(None), 'spam') + coro.close() + + def test_async_def_coroutines(self): + async def bar(): + return 'spam' + async def foo(): + return await bar() + + # production mode + data = self.loop.run_until_complete(foo()) + self.assertEqual(data, 'spam') + + # debug mode + self.loop.set_debug(True) + data = self.loop.run_until_complete(foo()) + self.assertEqual(data, 'spam') + + @mock.patch('asyncio.coroutines.logger') + def test_async_def_wrapped(self, m_log): + async def foo(): + pass + async def start(): + foo_coro = foo() + self.assertRegex( + repr(foo_coro), + r'') + + with support.check_warnings((r'.*foo.*was never', + RuntimeWarning)): + foo_coro = None + support.gc_collect() + self.assertTrue(m_log.error.called) + message = m_log.error.call_args[0][0] + self.assertRegex(message, + r'CoroWrapper.*foo.*was never') + + self.loop.set_debug(True) + self.loop.run_until_complete(start()) + + async def start(): + foo_coro = foo() + task = asyncio.ensure_future(foo_coro, loop=self.loop) + self.assertRegex(repr(task), r'Task.*foo.*running') + + self.loop.run_until_complete(start()) + + + def test_types_coroutine(self): + def gen(): + yield from () + return 'spam' + + @types.coroutine + def func(): + return gen() + + async def coro(): + wrapper = func() + self.assertIsInstance(wrapper, types._GeneratorWrapper) + return await wrapper + + data = self.loop.run_until_complete(coro()) + self.assertEqual(data, 'spam') + + def test_task_print_stack(self): + T = None + + async def foo(): + f = T.get_stack(limit=1) + try: + self.assertEqual(f[0].f_code.co_name, 'foo') + finally: + f = None + + async def runner(): + nonlocal T + T = asyncio.ensure_future(foo(), loop=self.loop) + await T + + self.loop.run_until_complete(runner()) + + def test_double_await(self): + async def afunc(): + await asyncio.sleep(0.1, loop=self.loop) + + async def runner(): + coro = afunc() + t = asyncio.Task(coro, loop=self.loop) + try: + await asyncio.sleep(0, loop=self.loop) + await coro + finally: + t.cancel() + + self.loop.set_debug(True) + with self.assertRaisesRegex( + RuntimeError, + r'Cannot await.*test_double_await.*\bafunc\b.*while.*\bsleep\b'): + + self.loop.run_until_complete(runner()) + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_proactor_events.py b/thirdparty/asyncio/tests/test_proactor_events.py new file mode 100644 index 0000000..4dfc612 --- /dev/null +++ b/thirdparty/asyncio/tests/test_proactor_events.py @@ -0,0 +1,594 @@ +"""Tests for proactor_events.py""" + +import socket +import unittest +from unittest import mock + +import asyncio +from asyncio.proactor_events import BaseProactorEventLoop +from asyncio.proactor_events import _ProactorSocketTransport +from asyncio.proactor_events import _ProactorWritePipeTransport +from asyncio.proactor_events import _ProactorDuplexPipeTransport +from asyncio import test_utils + + +def close_transport(transport): + # Don't call transport.close() because the event loop and the IOCP proactor + # are mocked + if transport._sock is None: + return + transport._sock.close() + transport._sock = None + + +class ProactorSocketTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + self.proactor = mock.Mock() + self.loop._proactor = self.proactor + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.sock = mock.Mock(socket.socket) + + def socket_transport(self, waiter=None): + transport = _ProactorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + fut = asyncio.Future(loop=self.loop) + tr = self.socket_transport(waiter=fut) + test_utils.run_briefly(self.loop) + self.assertIsNone(fut.result()) + self.protocol.connection_made(tr) + self.proactor.recv.assert_called_with(self.sock, 4096) + + def test_loop_reading(self): + tr = self.socket_transport() + tr._loop_reading() + self.loop._proactor.recv.assert_called_with(self.sock, 4096) + self.assertFalse(self.protocol.data_received.called) + self.assertFalse(self.protocol.eof_received.called) + + def test_loop_reading_data(self): + res = asyncio.Future(loop=self.loop) + res.set_result(b'data') + + tr = self.socket_transport() + tr._read_fut = res + tr._loop_reading(res) + self.loop._proactor.recv.assert_called_with(self.sock, 4096) + self.protocol.data_received.assert_called_with(b'data') + + def test_loop_reading_no_data(self): + res = asyncio.Future(loop=self.loop) + res.set_result(b'') + + tr = self.socket_transport() + self.assertRaises(AssertionError, tr._loop_reading, res) + + tr.close = mock.Mock() + tr._read_fut = res + tr._loop_reading(res) + self.assertFalse(self.loop._proactor.recv.called) + self.assertTrue(self.protocol.eof_received.called) + self.assertTrue(tr.close.called) + + def test_loop_reading_aborted(self): + err = self.loop._proactor.recv.side_effect = ConnectionAbortedError() + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._loop_reading() + tr._fatal_error.assert_called_with( + err, + 'Fatal read error on pipe transport') + + def test_loop_reading_aborted_closing(self): + self.loop._proactor.recv.side_effect = ConnectionAbortedError() + + tr = self.socket_transport() + tr._closing = True + tr._fatal_error = mock.Mock() + tr._loop_reading() + self.assertFalse(tr._fatal_error.called) + + def test_loop_reading_aborted_is_fatal(self): + self.loop._proactor.recv.side_effect = ConnectionAbortedError() + tr = self.socket_transport() + tr._closing = False + tr._fatal_error = mock.Mock() + tr._loop_reading() + self.assertTrue(tr._fatal_error.called) + + def test_loop_reading_conn_reset_lost(self): + err = self.loop._proactor.recv.side_effect = ConnectionResetError() + + tr = self.socket_transport() + tr._closing = False + tr._fatal_error = mock.Mock() + tr._force_close = mock.Mock() + tr._loop_reading() + self.assertFalse(tr._fatal_error.called) + tr._force_close.assert_called_with(err) + + def test_loop_reading_exception(self): + err = self.loop._proactor.recv.side_effect = (OSError()) + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._loop_reading() + tr._fatal_error.assert_called_with( + err, + 'Fatal read error on pipe transport') + + def test_write(self): + tr = self.socket_transport() + tr._loop_writing = mock.Mock() + tr.write(b'data') + self.assertEqual(tr._buffer, None) + tr._loop_writing.assert_called_with(data=b'data') + + def test_write_no_data(self): + tr = self.socket_transport() + tr.write(b'') + self.assertFalse(tr._buffer) + + def test_write_more(self): + tr = self.socket_transport() + tr._write_fut = mock.Mock() + tr._loop_writing = mock.Mock() + tr.write(b'data') + self.assertEqual(tr._buffer, b'data') + self.assertFalse(tr._loop_writing.called) + + def test_loop_writing(self): + tr = self.socket_transport() + tr._buffer = bytearray(b'data') + tr._loop_writing() + self.loop._proactor.send.assert_called_with(self.sock, b'data') + self.loop._proactor.send.return_value.add_done_callback.\ + assert_called_with(tr._loop_writing) + + @mock.patch('asyncio.proactor_events.logger') + def test_loop_writing_err(self, m_log): + err = self.loop._proactor.send.side_effect = OSError() + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._buffer = [b'da', b'ta'] + tr._loop_writing() + tr._fatal_error.assert_called_with( + err, + 'Fatal write error on pipe transport') + tr._conn_lost = 1 + + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + self.assertEqual(tr._buffer, None) + m_log.warning.assert_called_with('socket.send() raised exception.') + + def test_loop_writing_stop(self): + fut = asyncio.Future(loop=self.loop) + fut.set_result(b'data') + + tr = self.socket_transport() + tr._write_fut = fut + tr._loop_writing(fut) + self.assertIsNone(tr._write_fut) + + def test_loop_writing_closing(self): + fut = asyncio.Future(loop=self.loop) + fut.set_result(1) + + tr = self.socket_transport() + tr._write_fut = fut + tr.close() + tr._loop_writing(fut) + self.assertIsNone(tr._write_fut) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test_abort(self): + tr = self.socket_transport() + tr._force_close = mock.Mock() + tr.abort() + tr._force_close.assert_called_with(None) + + def test_close(self): + tr = self.socket_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertTrue(tr.is_closing()) + self.assertEqual(tr._conn_lost, 1) + + self.protocol.connection_lost.reset_mock() + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_close_write_fut(self): + tr = self.socket_transport() + tr._write_fut = mock.Mock() + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_close_buffer(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + @mock.patch('asyncio.base_events.logger') + def test_fatal_error(self, m_logging): + tr = self.socket_transport() + tr._force_close = mock.Mock() + tr._fatal_error(None) + self.assertTrue(tr._force_close.called) + self.assertTrue(m_logging.error.called) + + def test_force_close(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + read_fut = tr._read_fut = mock.Mock() + write_fut = tr._write_fut = mock.Mock() + tr._force_close(None) + + read_fut.cancel.assert_called_with() + write_fut.cancel.assert_called_with() + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertEqual(None, tr._buffer) + self.assertEqual(tr._conn_lost, 1) + + def test_force_close_idempotent(self): + tr = self.socket_transport() + tr._closing = True + tr._force_close(None) + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_fatal_error_2(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + tr._force_close(None) + + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertEqual(None, tr._buffer) + + def test_call_connection_lost(self): + tr = self.socket_transport() + tr._call_connection_lost(None) + self.assertTrue(self.protocol.connection_lost.called) + self.assertTrue(self.sock.close.called) + + def test_write_eof(self): + tr = self.socket_transport() + self.assertTrue(tr.can_write_eof()) + tr.write_eof() + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.write_eof() + self.assertEqual(self.sock.shutdown.call_count, 1) + tr.close() + + def test_write_eof_buffer(self): + tr = self.socket_transport() + f = asyncio.Future(loop=self.loop) + tr._loop._proactor.send.return_value = f + tr.write(b'data') + tr.write_eof() + self.assertTrue(tr._eof_written) + self.assertFalse(self.sock.shutdown.called) + tr._loop._proactor.send.assert_called_with(self.sock, b'data') + f.set_result(4) + self.loop._run_once() + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.close() + + def test_write_eof_write_pipe(self): + tr = _ProactorWritePipeTransport( + self.loop, self.sock, self.protocol) + self.assertTrue(tr.can_write_eof()) + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.loop._run_once() + self.assertTrue(self.sock.close.called) + tr.close() + + def test_write_eof_buffer_write_pipe(self): + tr = _ProactorWritePipeTransport(self.loop, self.sock, self.protocol) + f = asyncio.Future(loop=self.loop) + tr._loop._proactor.send.return_value = f + tr.write(b'data') + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.sock.shutdown.called) + tr._loop._proactor.send.assert_called_with(self.sock, b'data') + f.set_result(4) + self.loop._run_once() + self.loop._run_once() + self.assertTrue(self.sock.close.called) + tr.close() + + def test_write_eof_duplex_pipe(self): + tr = _ProactorDuplexPipeTransport( + self.loop, self.sock, self.protocol) + self.assertFalse(tr.can_write_eof()) + with self.assertRaises(NotImplementedError): + tr.write_eof() + close_transport(tr) + + def test_pause_resume_reading(self): + tr = self.socket_transport() + futures = [] + for msg in [b'data1', b'data2', b'data3', b'data4', b'']: + f = asyncio.Future(loop=self.loop) + f.set_result(msg) + futures.append(f) + self.loop._proactor.recv.side_effect = futures + self.loop._run_once() + self.assertFalse(tr._paused) + self.loop._run_once() + self.protocol.data_received.assert_called_with(b'data1') + self.loop._run_once() + self.protocol.data_received.assert_called_with(b'data2') + tr.pause_reading() + self.assertTrue(tr._paused) + for i in range(10): + self.loop._run_once() + self.protocol.data_received.assert_called_with(b'data2') + tr.resume_reading() + self.assertFalse(tr._paused) + self.loop._run_once() + self.protocol.data_received.assert_called_with(b'data3') + self.loop._run_once() + self.protocol.data_received.assert_called_with(b'data4') + tr.close() + + + def pause_writing_transport(self, high): + tr = self.socket_transport() + tr.set_write_buffer_limits(high=high) + + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertFalse(self.protocol.pause_writing.called) + self.assertFalse(self.protocol.resume_writing.called) + return tr + + def test_pause_resume_writing(self): + tr = self.pause_writing_transport(high=4) + + # write a large chunk, must pause writing + fut = asyncio.Future(loop=self.loop) + self.loop._proactor.send.return_value = fut + tr.write(b'large data') + self.loop._run_once() + self.assertTrue(self.protocol.pause_writing.called) + + # flush the buffer + fut.set_result(None) + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertTrue(self.protocol.resume_writing.called) + + def test_pause_writing_2write(self): + tr = self.pause_writing_transport(high=4) + + # first short write, the buffer is not full (3 <= 4) + fut1 = asyncio.Future(loop=self.loop) + self.loop._proactor.send.return_value = fut1 + tr.write(b'123') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 3) + self.assertFalse(self.protocol.pause_writing.called) + + # fill the buffer, must pause writing (6 > 4) + tr.write(b'abc') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 6) + self.assertTrue(self.protocol.pause_writing.called) + + def test_pause_writing_3write(self): + tr = self.pause_writing_transport(high=4) + + # first short write, the buffer is not full (1 <= 4) + fut = asyncio.Future(loop=self.loop) + self.loop._proactor.send.return_value = fut + tr.write(b'1') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 1) + self.assertFalse(self.protocol.pause_writing.called) + + # second short write, the buffer is not full (3 <= 4) + tr.write(b'23') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 3) + self.assertFalse(self.protocol.pause_writing.called) + + # fill the buffer, must pause writing (6 > 4) + tr.write(b'abc') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 6) + self.assertTrue(self.protocol.pause_writing.called) + + def test_dont_pause_writing(self): + tr = self.pause_writing_transport(high=4) + + # write a large chunk which completes immedialty, + # it should not pause writing + fut = asyncio.Future(loop=self.loop) + fut.set_result(None) + self.loop._proactor.send.return_value = fut + tr.write(b'very large data') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertFalse(self.protocol.pause_writing.called) + + +class BaseProactorEventLoopTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + + self.sock = test_utils.mock_nonblocking_socket() + self.proactor = mock.Mock() + + self.ssock, self.csock = mock.Mock(), mock.Mock() + + class EventLoop(BaseProactorEventLoop): + def _socketpair(s): + return (self.ssock, self.csock) + + self.loop = EventLoop(self.proactor) + self.set_event_loop(self.loop) + + @mock.patch.object(BaseProactorEventLoop, 'call_soon') + @mock.patch.object(BaseProactorEventLoop, '_socketpair') + def test_ctor(self, socketpair, call_soon): + ssock, csock = socketpair.return_value = ( + mock.Mock(), mock.Mock()) + loop = BaseProactorEventLoop(self.proactor) + self.assertIs(loop._ssock, ssock) + self.assertIs(loop._csock, csock) + self.assertEqual(loop._internal_fds, 1) + call_soon.assert_called_with(loop._loop_self_reading) + loop.close() + + def test_close_self_pipe(self): + self.loop._close_self_pipe() + self.assertEqual(self.loop._internal_fds, 0) + self.assertTrue(self.ssock.close.called) + self.assertTrue(self.csock.close.called) + self.assertIsNone(self.loop._ssock) + self.assertIsNone(self.loop._csock) + + # Don't call close(): _close_self_pipe() cannot be called twice + self.loop._closed = True + + def test_close(self): + self.loop._close_self_pipe = mock.Mock() + self.loop.close() + self.assertTrue(self.loop._close_self_pipe.called) + self.assertTrue(self.proactor.close.called) + self.assertIsNone(self.loop._proactor) + + self.loop._close_self_pipe.reset_mock() + self.loop.close() + self.assertFalse(self.loop._close_self_pipe.called) + + def test_sock_recv(self): + self.loop.sock_recv(self.sock, 1024) + self.proactor.recv.assert_called_with(self.sock, 1024) + + def test_sock_sendall(self): + self.loop.sock_sendall(self.sock, b'data') + self.proactor.send.assert_called_with(self.sock, b'data') + + def test_sock_connect(self): + self.loop.sock_connect(self.sock, ('1.2.3.4', 123)) + self.proactor.connect.assert_called_with(self.sock, ('1.2.3.4', 123)) + + def test_sock_accept(self): + self.loop.sock_accept(self.sock) + self.proactor.accept.assert_called_with(self.sock) + + def test_socketpair(self): + class EventLoop(BaseProactorEventLoop): + # override the destructor to not log a ResourceWarning + def __del__(self): + pass + self.assertRaises( + NotImplementedError, EventLoop, self.proactor) + + def test_make_socket_transport(self): + tr = self.loop._make_socket_transport(self.sock, asyncio.Protocol()) + self.assertIsInstance(tr, _ProactorSocketTransport) + close_transport(tr) + + def test_loop_self_reading(self): + self.loop._loop_self_reading() + self.proactor.recv.assert_called_with(self.ssock, 4096) + self.proactor.recv.return_value.add_done_callback.assert_called_with( + self.loop._loop_self_reading) + + def test_loop_self_reading_fut(self): + fut = mock.Mock() + self.loop._loop_self_reading(fut) + self.assertTrue(fut.result.called) + self.proactor.recv.assert_called_with(self.ssock, 4096) + self.proactor.recv.return_value.add_done_callback.assert_called_with( + self.loop._loop_self_reading) + + def test_loop_self_reading_exception(self): + self.loop.close = mock.Mock() + self.loop.call_exception_handler = mock.Mock() + self.proactor.recv.side_effect = OSError() + self.loop._loop_self_reading() + self.assertTrue(self.loop.call_exception_handler.called) + + def test_write_to_self(self): + self.loop._write_to_self() + self.csock.send.assert_called_with(b'\0') + + def test_process_events(self): + self.loop._process_events([]) + + @mock.patch('asyncio.base_events.logger') + def test_create_server(self, m_log): + pf = mock.Mock() + call_soon = self.loop.call_soon = mock.Mock() + + self.loop._start_serving(pf, self.sock) + self.assertTrue(call_soon.called) + + # callback + loop = call_soon.call_args[0][0] + loop() + self.proactor.accept.assert_called_with(self.sock) + + # conn + fut = mock.Mock() + fut.result.return_value = (mock.Mock(), mock.Mock()) + + make_tr = self.loop._make_socket_transport = mock.Mock() + loop(fut) + self.assertTrue(fut.result.called) + self.assertTrue(make_tr.called) + + # exception + fut.result.side_effect = OSError() + loop(fut) + self.assertTrue(self.sock.close.called) + self.assertTrue(m_log.error.called) + + def test_create_server_cancel(self): + pf = mock.Mock() + call_soon = self.loop.call_soon = mock.Mock() + + self.loop._start_serving(pf, self.sock) + loop = call_soon.call_args[0][0] + + # cancelled + fut = asyncio.Future(loop=self.loop) + fut.cancel() + loop(fut) + self.assertTrue(self.sock.close.called) + + def test_stop_serving(self): + sock = mock.Mock() + self.loop._stop_serving(sock) + self.assertTrue(sock.close.called) + self.proactor._stop_serving.assert_called_with(sock) + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_queues.py b/thirdparty/asyncio/tests/test_queues.py new file mode 100644 index 0000000..fe5a6db --- /dev/null +++ b/thirdparty/asyncio/tests/test_queues.py @@ -0,0 +1,626 @@ +"""Tests for queues.py""" + +import unittest +from unittest import mock + +import asyncio +from asyncio import test_utils + + +class _QueueTestBase(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + +class QueueBasicTests(_QueueTestBase): + + def _test_repr_or_str(self, fn, expect_id): + """Test Queue's repr or str. + + fn is repr or str. expect_id is True if we expect the Queue's id to + appear in fn(Queue()). + """ + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0.1 + self.assertAlmostEqual(0.2, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + q = asyncio.Queue(loop=loop) + self.assertTrue(fn(q).startswith('", repr(stream)) + + def test___repr__nondefault_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=123) + self.assertEqual("", repr(stream)) + + def test___repr__eof(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_eof() + self.assertEqual("", repr(stream)) + + def test___repr__data(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + self.assertEqual("", repr(stream)) + + def test___repr__exception(self): + stream = asyncio.StreamReader(loop=self.loop) + exc = RuntimeError() + stream.set_exception(exc) + self.assertEqual("", repr(stream)) + + def test___repr__waiter(self): + stream = asyncio.StreamReader(loop=self.loop) + stream._waiter = asyncio.Future(loop=self.loop) + self.assertRegex( + repr(stream), + r">") + stream._waiter.set_result(None) + self.loop.run_until_complete(stream._waiter) + stream._waiter = None + self.assertEqual("", repr(stream)) + + def test___repr__transport(self): + stream = asyncio.StreamReader(loop=self.loop) + stream._transport = mock.Mock() + stream._transport.__repr__ = mock.Mock() + stream._transport.__repr__.return_value = "" + self.assertEqual(">", repr(stream)) + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_subprocess.py b/thirdparty/asyncio/tests/test_subprocess.py new file mode 100644 index 0000000..bba688b --- /dev/null +++ b/thirdparty/asyncio/tests/test_subprocess.py @@ -0,0 +1,501 @@ +import signal +import sys +import unittest +import warnings +from unittest import mock + +import asyncio +from asyncio import base_subprocess +from asyncio import subprocess +from asyncio import test_utils +try: + from test import support +except ImportError: + from asyncio import test_support as support +if sys.platform != 'win32': + from asyncio import unix_events + +# Program blocking +PROGRAM_BLOCKED = [sys.executable, '-c', 'import time; time.sleep(3600)'] + +# Program copying input to output +PROGRAM_CAT = [ + sys.executable, '-c', + ';'.join(('import sys', + 'data = sys.stdin.buffer.read()', + 'sys.stdout.buffer.write(data)'))] + +class TestSubprocessTransport(base_subprocess.BaseSubprocessTransport): + def _start(self, *args, **kwargs): + self._proc = mock.Mock() + self._proc.stdin = None + self._proc.stdout = None + self._proc.stderr = None + + +class SubprocessTransportTests(test_utils.TestCase): + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.set_event_loop(self.loop) + + + def create_transport(self, waiter=None): + protocol = mock.Mock() + protocol.connection_made._is_coroutine = False + protocol.process_exited._is_coroutine = False + transport = TestSubprocessTransport( + self.loop, protocol, ['test'], False, + None, None, None, 0, waiter=waiter) + return (transport, protocol) + + def test_proc_exited(self): + waiter = asyncio.Future(loop=self.loop) + transport, protocol = self.create_transport(waiter) + transport._process_exited(6) + self.loop.run_until_complete(waiter) + + self.assertEqual(transport.get_returncode(), 6) + + self.assertTrue(protocol.connection_made.called) + self.assertTrue(protocol.process_exited.called) + self.assertTrue(protocol.connection_lost.called) + self.assertEqual(protocol.connection_lost.call_args[0], (None,)) + + self.assertFalse(transport.is_closing()) + self.assertIsNone(transport._loop) + self.assertIsNone(transport._proc) + self.assertIsNone(transport._protocol) + + # methods must raise ProcessLookupError if the process exited + self.assertRaises(ProcessLookupError, + transport.send_signal, signal.SIGTERM) + self.assertRaises(ProcessLookupError, transport.terminate) + self.assertRaises(ProcessLookupError, transport.kill) + + transport.close() + + +class SubprocessMixin: + + def test_stdin_stdout(self): + args = PROGRAM_CAT + + @asyncio.coroutine + def run(data): + proc = yield from asyncio.create_subprocess_exec( + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + loop=self.loop) + + # feed data + proc.stdin.write(data) + yield from proc.stdin.drain() + proc.stdin.close() + + # get output and exitcode + data = yield from proc.stdout.read() + exitcode = yield from proc.wait() + return (exitcode, data) + + task = run(b'some data') + task = asyncio.wait_for(task, 60.0, loop=self.loop) + exitcode, stdout = self.loop.run_until_complete(task) + self.assertEqual(exitcode, 0) + self.assertEqual(stdout, b'some data') + + def test_communicate(self): + args = PROGRAM_CAT + + @asyncio.coroutine + def run(data): + proc = yield from asyncio.create_subprocess_exec( + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + loop=self.loop) + stdout, stderr = yield from proc.communicate(data) + return proc.returncode, stdout + + task = run(b'some data') + task = asyncio.wait_for(task, 60.0, loop=self.loop) + exitcode, stdout = self.loop.run_until_complete(task) + self.assertEqual(exitcode, 0) + self.assertEqual(stdout, b'some data') + + def test_shell(self): + create = asyncio.create_subprocess_shell('exit 7', + loop=self.loop) + proc = self.loop.run_until_complete(create) + exitcode = self.loop.run_until_complete(proc.wait()) + self.assertEqual(exitcode, 7) + + def test_start_new_session(self): + # start the new process in a new session + create = asyncio.create_subprocess_shell('exit 8', + start_new_session=True, + loop=self.loop) + proc = self.loop.run_until_complete(create) + exitcode = self.loop.run_until_complete(proc.wait()) + self.assertEqual(exitcode, 8) + + def test_kill(self): + args = PROGRAM_BLOCKED + create = asyncio.create_subprocess_exec(*args, loop=self.loop) + proc = self.loop.run_until_complete(create) + proc.kill() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + + def test_terminate(self): + args = PROGRAM_BLOCKED + create = asyncio.create_subprocess_exec(*args, loop=self.loop) + proc = self.loop.run_until_complete(create) + proc.terminate() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGTERM, returncode) + + @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + def test_send_signal(self): + code = 'import time; print("sleeping", flush=True); time.sleep(3600)' + args = [sys.executable, '-c', code] + create = asyncio.create_subprocess_exec(*args, + stdout=subprocess.PIPE, + loop=self.loop) + proc = self.loop.run_until_complete(create) + + @asyncio.coroutine + def send_signal(proc): + # basic synchronization to wait until the program is sleeping + line = yield from proc.stdout.readline() + self.assertEqual(line, b'sleeping\n') + + proc.send_signal(signal.SIGHUP) + returncode = (yield from proc.wait()) + return returncode + + returncode = self.loop.run_until_complete(send_signal(proc)) + self.assertEqual(-signal.SIGHUP, returncode) + + def prepare_broken_pipe_test(self): + # buffer large enough to feed the whole pipe buffer + large_data = b'x' * support.PIPE_MAX_SIZE + + # the program ends before the stdin can be feeded + create = asyncio.create_subprocess_exec( + sys.executable, '-c', 'pass', + stdin=subprocess.PIPE, + loop=self.loop) + proc = self.loop.run_until_complete(create) + return (proc, large_data) + + def test_stdin_broken_pipe(self): + proc, large_data = self.prepare_broken_pipe_test() + + @asyncio.coroutine + def write_stdin(proc, data): + proc.stdin.write(data) + yield from proc.stdin.drain() + + coro = write_stdin(proc, large_data) + # drain() must raise BrokenPipeError or ConnectionResetError + with test_utils.disable_logger(): + self.assertRaises((BrokenPipeError, ConnectionResetError), + self.loop.run_until_complete, coro) + self.loop.run_until_complete(proc.wait()) + + def test_communicate_ignore_broken_pipe(self): + proc, large_data = self.prepare_broken_pipe_test() + + # communicate() must ignore BrokenPipeError when feeding stdin + with test_utils.disable_logger(): + self.loop.run_until_complete(proc.communicate(large_data)) + self.loop.run_until_complete(proc.wait()) + + def test_pause_reading(self): + limit = 10 + size = (limit * 2 + 1) + + @asyncio.coroutine + def test_pause_reading(): + code = '\n'.join(( + 'import sys', + 'sys.stdout.write("x" * %s)' % size, + 'sys.stdout.flush()', + )) + + connect_read_pipe = self.loop.connect_read_pipe + + @asyncio.coroutine + def connect_read_pipe_mock(*args, **kw): + transport, protocol = yield from connect_read_pipe(*args, **kw) + transport.pause_reading = mock.Mock() + transport.resume_reading = mock.Mock() + return (transport, protocol) + + self.loop.connect_read_pipe = connect_read_pipe_mock + + proc = yield from asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + limit=limit, + loop=self.loop) + stdout_transport = proc._transport.get_pipe_transport(1) + + stdout, stderr = yield from proc.communicate() + + # The child process produced more than limit bytes of output, + # the stream reader transport should pause the protocol to not + # allocate too much memory. + return (stdout, stdout_transport) + + # Issue #22685: Ensure that the stream reader pauses the protocol + # when the child process produces too much data + stdout, transport = self.loop.run_until_complete(test_pause_reading()) + + self.assertEqual(stdout, b'x' * size) + self.assertTrue(transport.pause_reading.called) + self.assertTrue(transport.resume_reading.called) + + def test_stdin_not_inheritable(self): + # asyncio issue #209: stdin must not be inheritable, otherwise + # the Process.communicate() hangs + @asyncio.coroutine + def len_message(message): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = yield from asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + loop=self.loop) + stdout, stderr = yield from proc.communicate(message) + exitcode = yield from proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(len_message(b'abc')) + self.assertEqual(output.rstrip(), b'3') + self.assertEqual(exitcode, 0) + + def test_empty_input(self): + @asyncio.coroutine + def empty_input(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = yield from asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + loop=self.loop) + stdout, stderr = yield from proc.communicate(b'') + exitcode = yield from proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_input()) + self.assertEqual(output.rstrip(), b'0') + self.assertEqual(exitcode, 0) + + def test_cancel_process_wait(self): + # Issue #23140: cancel Process.wait() + + @asyncio.coroutine + def cancel_wait(): + proc = yield from asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + loop=self.loop) + + # Create an internal future waiting on the process exit + task = self.loop.create_task(proc.wait()) + self.loop.call_soon(task.cancel) + try: + yield from task + except asyncio.CancelledError: + pass + + # Cancel the future + task.cancel() + + # Kill the process and wait until it is done + proc.kill() + yield from proc.wait() + + self.loop.run_until_complete(cancel_wait()) + + def test_cancel_make_subprocess_transport_exec(self): + @asyncio.coroutine + def cancel_make_transport(): + coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, + loop=self.loop) + task = self.loop.create_task(coro) + + self.loop.call_soon(task.cancel) + try: + yield from task + except asyncio.CancelledError: + pass + + # ignore the log: + # "Exception during subprocess creation, kill the subprocess" + with test_utils.disable_logger(): + self.loop.run_until_complete(cancel_make_transport()) + + def test_cancel_post_init(self): + @asyncio.coroutine + def cancel_make_transport(): + coro = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + task = self.loop.create_task(coro) + + self.loop.call_soon(task.cancel) + try: + yield from task + except asyncio.CancelledError: + pass + + # ignore the log: + # "Exception during subprocess creation, kill the subprocess" + with test_utils.disable_logger(): + self.loop.run_until_complete(cancel_make_transport()) + test_utils.run_briefly(self.loop) + + def test_close_kill_running(self): + @asyncio.coroutine + def kill_running(): + create = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + transport, protocol = yield from create + + kill_called = False + def kill(): + nonlocal kill_called + kill_called = True + orig_kill() + + proc = transport.get_extra_info('subprocess') + orig_kill = proc.kill + proc.kill = kill + returncode = transport.get_returncode() + transport.close() + yield from transport._wait() + return (returncode, kill_called) + + # Ignore "Close running child process: kill ..." log + with test_utils.disable_logger(): + returncode, killed = self.loop.run_until_complete(kill_running()) + self.assertIsNone(returncode) + + # transport.close() must kill the process if it is still running + self.assertTrue(killed) + test_utils.run_briefly(self.loop) + + def test_close_dont_kill_finished(self): + @asyncio.coroutine + def kill_running(): + create = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + transport, protocol = yield from create + proc = transport.get_extra_info('subprocess') + + # kill the process (but asyncio is not notified immediately) + proc.kill() + proc.wait() + + proc.kill = mock.Mock() + proc_returncode = proc.poll() + transport_returncode = transport.get_returncode() + transport.close() + return (proc_returncode, transport_returncode, proc.kill.called) + + # Ignore "Unknown child process pid ..." log of SafeChildWatcher, + # emitted because the test already consumes the exit status: + # proc.wait() + with test_utils.disable_logger(): + result = self.loop.run_until_complete(kill_running()) + test_utils.run_briefly(self.loop) + + proc_returncode, transport_return_code, killed = result + + self.assertIsNotNone(proc_returncode) + self.assertIsNone(transport_return_code) + + # transport.close() must not kill the process if it finished, even if + # the transport was not notified yet + self.assertFalse(killed) + + # Unlike SafeChildWatcher, FastChildWatcher does not pop the + # callbacks if waitpid() is called elsewhere. Let's clear them + # manually to avoid a warning when the watcher is detached. + if sys.platform != 'win32' and \ + isinstance(self, SubprocessFastWatcherTests): + asyncio.get_child_watcher()._callbacks.clear() + + def test_popen_error(self): + # Issue #24763: check that the subprocess transport is closed + # when BaseSubprocessTransport fails + if sys.platform == 'win32': + target = 'asyncio.windows_utils.Popen' + else: + target = 'subprocess.Popen' + with mock.patch(target) as popen: + exc = ZeroDivisionError + popen.side_effect = exc + + create = asyncio.create_subprocess_exec(sys.executable, '-c', + 'pass', loop=self.loop) + with warnings.catch_warnings(record=True) as warns: + with self.assertRaises(exc): + self.loop.run_until_complete(create) + self.assertEqual(warns, []) + + +if sys.platform != 'win32': + # Unix + class SubprocessWatcherMixin(SubprocessMixin): + + Watcher = None + + def setUp(self): + super().setUp() + policy = asyncio.get_event_loop_policy() + self.loop = policy.new_event_loop() + self.set_event_loop(self.loop) + + watcher = self.Watcher() + watcher.attach_loop(self.loop) + policy.set_child_watcher(watcher) + self.addCleanup(policy.set_child_watcher, None) + + class SubprocessSafeWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + + Watcher = unix_events.SafeChildWatcher + + class SubprocessFastWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + + Watcher = unix_events.FastChildWatcher + +else: + # Windows + class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.ProactorEventLoop() + self.set_event_loop(self.loop) + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_tasks.py b/thirdparty/asyncio/tests/test_tasks.py new file mode 100644 index 0000000..22accf5 --- /dev/null +++ b/thirdparty/asyncio/tests/test_tasks.py @@ -0,0 +1,2341 @@ +"""Tests for tasks.py.""" + +import contextlib +import functools +import io +import os +import re +import sys +import time +import types +import unittest +import weakref +from unittest import mock + +import asyncio +from asyncio import coroutines +from asyncio import test_utils +try: + from test import support +except ImportError: + from asyncio import test_support as support +try: + from test.support.script_helper import assert_python_ok +except ImportError: + try: + from test.script_helper import assert_python_ok + except ImportError: + from asyncio.test_support import assert_python_ok + + +PY34 = (sys.version_info >= (3, 4)) +PY35 = (sys.version_info >= (3, 5)) + + +@asyncio.coroutine +def coroutine_function(): + pass + + +@contextlib.contextmanager +def set_coroutine_debug(enabled): + coroutines = asyncio.coroutines + + old_debug = coroutines._DEBUG + try: + coroutines._DEBUG = enabled + yield + finally: + coroutines._DEBUG = old_debug + + + +def format_coroutine(qualname, state, src, source_traceback, generator=False): + if generator: + state = '%s' % state + else: + state = '%s, defined' % state + if source_traceback is not None: + frame = source_traceback[-1] + return ('coro=<%s() %s at %s> created at %s:%s' + % (qualname, state, src, frame[0], frame[1])) + else: + return 'coro=<%s() %s at %s>' % (qualname, state, src) + + +class Dummy: + + def __repr__(self): + return '' + + def __call__(self, *args): + pass + + +class TaskTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + def test_other_loop_future(self): + other_loop = asyncio.new_event_loop() + fut = asyncio.Future(loop=other_loop) + + @asyncio.coroutine + def run(fut): + yield from fut + + try: + with self.assertRaisesRegex(RuntimeError, + r'Task .* got Future .* attached'): + self.loop.run_until_complete(run(fut)) + finally: + other_loop.close() + + def test_task_awaits_on_itself(self): + @asyncio.coroutine + def test(): + yield from task + + task = asyncio.ensure_future(test(), loop=self.loop) + + with self.assertRaisesRegex(RuntimeError, + 'Task cannot await on itself'): + self.loop.run_until_complete(task) + + def test_task_class(self): + @asyncio.coroutine + def notmuch(): + return 'ok' + t = asyncio.Task(notmuch(), loop=self.loop) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + self.assertIs(t._loop, self.loop) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + t = asyncio.Task(notmuch(), loop=loop) + self.assertIs(t._loop, loop) + loop.run_until_complete(t) + loop.close() + + def test_ensure_future_coroutine(self): + @asyncio.coroutine + def notmuch(): + return 'ok' + t = asyncio.ensure_future(notmuch(), loop=self.loop) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + self.assertIs(t._loop, self.loop) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + t = asyncio.ensure_future(notmuch(), loop=loop) + self.assertIs(t._loop, loop) + loop.run_until_complete(t) + loop.close() + + def test_ensure_future_future(self): + f_orig = asyncio.Future(loop=self.loop) + f_orig.set_result('ko') + + f = asyncio.ensure_future(f_orig) + self.loop.run_until_complete(f) + self.assertTrue(f.done()) + self.assertEqual(f.result(), 'ko') + self.assertIs(f, f_orig) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + with self.assertRaises(ValueError): + f = asyncio.ensure_future(f_orig, loop=loop) + + loop.close() + + f = asyncio.ensure_future(f_orig, loop=self.loop) + self.assertIs(f, f_orig) + + def test_ensure_future_task(self): + @asyncio.coroutine + def notmuch(): + return 'ok' + t_orig = asyncio.Task(notmuch(), loop=self.loop) + t = asyncio.ensure_future(t_orig) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + self.assertIs(t, t_orig) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + with self.assertRaises(ValueError): + t = asyncio.ensure_future(t_orig, loop=loop) + + loop.close() + + t = asyncio.ensure_future(t_orig, loop=self.loop) + self.assertIs(t, t_orig) + + @unittest.skipUnless(PY35, 'need python 3.5 or later') + def test_ensure_future_awaitable(self): + class Aw: + def __init__(self, coro): + self.coro = coro + def __await__(self): + return (yield from self.coro) + + @asyncio.coroutine + def coro(): + return 'ok' + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + fut = asyncio.ensure_future(Aw(coro()), loop=loop) + loop.run_until_complete(fut) + assert fut.result() == 'ok' + + def test_ensure_future_neither(self): + with self.assertRaises(TypeError): + asyncio.ensure_future('ok') + + def test_async_warning(self): + f = asyncio.Future(loop=self.loop) + with self.assertWarnsRegex(DeprecationWarning, + 'function is deprecated, use ensure_'): + self.assertIs(f, asyncio.async(f)) + + def test_get_stack(self): + T = None + + @asyncio.coroutine + def foo(): + yield from bar() + + @asyncio.coroutine + def bar(): + # test get_stack() + f = T.get_stack(limit=1) + try: + self.assertEqual(f[0].f_code.co_name, 'foo') + finally: + f = None + + # test print_stack() + file = io.StringIO() + T.print_stack(limit=1, file=file) + file.seek(0) + tb = file.read() + self.assertRegex(tb, r'foo\(\) running') + + @asyncio.coroutine + def runner(): + nonlocal T + T = asyncio.ensure_future(foo(), loop=self.loop) + yield from T + + self.loop.run_until_complete(runner()) + + def test_task_repr(self): + self.loop.set_debug(False) + + @asyncio.coroutine + def notmuch(): + yield from [] + return 'abc' + + # test coroutine function + self.assertEqual(notmuch.__name__, 'notmuch') + if PY35: + self.assertEqual(notmuch.__qualname__, + 'TaskTests.test_task_repr..notmuch') + self.assertEqual(notmuch.__module__, __name__) + + filename, lineno = test_utils.get_function_source(notmuch) + src = "%s:%s" % (filename, lineno) + + # test coroutine object + gen = notmuch() + if coroutines._DEBUG or PY35: + coro_qualname = 'TaskTests.test_task_repr..notmuch' + else: + coro_qualname = 'notmuch' + self.assertEqual(gen.__name__, 'notmuch') + if PY35: + self.assertEqual(gen.__qualname__, + coro_qualname) + + # test pending Task + t = asyncio.Task(gen, loop=self.loop) + t.add_done_callback(Dummy()) + + coro = format_coroutine(coro_qualname, 'running', src, + t._source_traceback, generator=True) + self.assertEqual(repr(t), + '()]>' % coro) + + # test cancelling Task + t.cancel() # Does not take immediate effect! + self.assertEqual(repr(t), + '()]>' % coro) + + # test cancelled Task + self.assertRaises(asyncio.CancelledError, + self.loop.run_until_complete, t) + coro = format_coroutine(coro_qualname, 'done', src, + t._source_traceback) + self.assertEqual(repr(t), + '' % coro) + + # test finished Task + t = asyncio.Task(notmuch(), loop=self.loop) + self.loop.run_until_complete(t) + coro = format_coroutine(coro_qualname, 'done', src, + t._source_traceback) + self.assertEqual(repr(t), + "" % coro) + + def test_task_repr_coro_decorator(self): + self.loop.set_debug(False) + + @asyncio.coroutine + def notmuch(): + # notmuch() function doesn't use yield from: it will be wrapped by + # @coroutine decorator + return 123 + + # test coroutine function + self.assertEqual(notmuch.__name__, 'notmuch') + if PY35: + self.assertEqual(notmuch.__qualname__, + 'TaskTests.test_task_repr_coro_decorator' + '..notmuch') + self.assertEqual(notmuch.__module__, __name__) + + # test coroutine object + gen = notmuch() + if coroutines._DEBUG or PY35: + # On Python >= 3.5, generators now inherit the name of the + # function, as expected, and have a qualified name (__qualname__ + # attribute). + coro_name = 'notmuch' + coro_qualname = ('TaskTests.test_task_repr_coro_decorator' + '..notmuch') + else: + # On Python < 3.5, generators inherit the name of the code, not of + # the function. See: http://bugs.python.org/issue21205 + coro_name = coro_qualname = 'coro' + self.assertEqual(gen.__name__, coro_name) + if PY35: + self.assertEqual(gen.__qualname__, coro_qualname) + + # test repr(CoroWrapper) + if coroutines._DEBUG: + # format the coroutine object + if coroutines._DEBUG: + filename, lineno = test_utils.get_function_source(notmuch) + frame = gen._source_traceback[-1] + coro = ('%s() running, defined at %s:%s, created at %s:%s' + % (coro_qualname, filename, lineno, + frame[0], frame[1])) + else: + code = gen.gi_code + coro = ('%s() running at %s:%s' + % (coro_qualname, code.co_filename, + code.co_firstlineno)) + + self.assertEqual(repr(gen), '' % coro) + + # test pending Task + t = asyncio.Task(gen, loop=self.loop) + t.add_done_callback(Dummy()) + + # format the coroutine object + if coroutines._DEBUG: + src = '%s:%s' % test_utils.get_function_source(notmuch) + else: + code = gen.gi_code + src = '%s:%s' % (code.co_filename, code.co_firstlineno) + coro = format_coroutine(coro_qualname, 'running', src, + t._source_traceback, + generator=not coroutines._DEBUG) + self.assertEqual(repr(t), + '()]>' % coro) + self.loop.run_until_complete(t) + + def test_task_repr_wait_for(self): + self.loop.set_debug(False) + + @asyncio.coroutine + def wait_for(fut): + return (yield from fut) + + fut = asyncio.Future(loop=self.loop) + task = asyncio.Task(wait_for(fut), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertRegex(repr(task), + '' % re.escape(repr(fut))) + + fut.set_result(None) + self.loop.run_until_complete(task) + + def test_task_repr_partial_corowrapper(self): + # Issue #222: repr(CoroWrapper) must not fail in debug mode if the + # coroutine is a partial function + with set_coroutine_debug(True): + self.loop.set_debug(True) + + @asyncio.coroutine + def func(x, y): + yield from asyncio.sleep(0) + + partial_func = asyncio.coroutine(functools.partial(func, 1)) + task = self.loop.create_task(partial_func(2)) + + # make warnings quiet + task._log_destroy_pending = False + self.addCleanup(task._coro.close) + + coro_repr = repr(task._coro) + expected = ('.func(1)() running, ') + self.assertTrue(coro_repr.startswith(expected), + coro_repr) + + def test_task_basics(self): + @asyncio.coroutine + def outer(): + a = yield from inner1() + b = yield from inner2() + return a+b + + @asyncio.coroutine + def inner1(): + return 42 + + @asyncio.coroutine + def inner2(): + return 1000 + + t = outer() + self.assertEqual(self.loop.run_until_complete(t), 1042) + + def test_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + @asyncio.coroutine + def task(): + yield from asyncio.sleep(10.0, loop=loop) + return 12 + + t = asyncio.Task(task(), loop=loop) + loop.call_soon(t.cancel) + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t.cancel()) + + def test_cancel_yield(self): + @asyncio.coroutine + def task(): + yield + yield + return 12 + + t = asyncio.Task(task(), loop=self.loop) + test_utils.run_briefly(self.loop) # start coro + t.cancel() + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t.cancel()) + + def test_cancel_inner_future(self): + f = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def task(): + yield from f + return 12 + + t = asyncio.Task(task(), loop=self.loop) + test_utils.run_briefly(self.loop) # start task + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t) + self.assertTrue(f.cancelled()) + self.assertTrue(t.cancelled()) + + def test_cancel_both_task_and_inner_future(self): + f = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def task(): + yield from f + return 12 + + t = asyncio.Task(task(), loop=self.loop) + test_utils.run_briefly(self.loop) + + f.cancel() + t.cancel() + + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t) + + self.assertTrue(t.done()) + self.assertTrue(f.cancelled()) + self.assertTrue(t.cancelled()) + + def test_cancel_task_catching(self): + fut1 = asyncio.Future(loop=self.loop) + fut2 = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def task(): + yield from fut1 + try: + yield from fut2 + except asyncio.CancelledError: + return 42 + + t = asyncio.Task(task(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut1) # White-box test. + fut1.set_result(None) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut2) # White-box test. + t.cancel() + self.assertTrue(fut2.cancelled()) + res = self.loop.run_until_complete(t) + self.assertEqual(res, 42) + self.assertFalse(t.cancelled()) + + def test_cancel_task_ignoring(self): + fut1 = asyncio.Future(loop=self.loop) + fut2 = asyncio.Future(loop=self.loop) + fut3 = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def task(): + yield from fut1 + try: + yield from fut2 + except asyncio.CancelledError: + pass + res = yield from fut3 + return res + + t = asyncio.Task(task(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut1) # White-box test. + fut1.set_result(None) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut2) # White-box test. + t.cancel() + self.assertTrue(fut2.cancelled()) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut3) # White-box test. + fut3.set_result(42) + res = self.loop.run_until_complete(t) + self.assertEqual(res, 42) + self.assertFalse(fut3.cancelled()) + self.assertFalse(t.cancelled()) + + def test_cancel_current_task(self): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + @asyncio.coroutine + def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + # The sleep should be cancelled immediately. + yield from asyncio.sleep(100, loop=loop) + return 12 + + t = asyncio.Task(task(), loop=loop) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + + def test_stop_while_run_in_complete(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0.1 + self.assertAlmostEqual(0.2, when) + when = yield 0.1 + self.assertAlmostEqual(0.3, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + x = 0 + waiters = [] + + @asyncio.coroutine + def task(): + nonlocal x + while x < 10: + waiters.append(asyncio.sleep(0.1, loop=loop)) + yield from waiters[-1] + x += 1 + if x == 2: + loop.stop() + + t = asyncio.Task(task(), loop=loop) + with self.assertRaises(RuntimeError) as cm: + loop.run_until_complete(t) + self.assertEqual(str(cm.exception), + 'Event loop stopped before Future completed.') + self.assertFalse(t.done()) + self.assertEqual(x, 2) + self.assertAlmostEqual(0.3, loop.time()) + + # close generators + for w in waiters: + w.close() + t.cancel() + self.assertRaises(asyncio.CancelledError, loop.run_until_complete, t) + + def test_wait_for(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.2, when) + when = yield 0 + self.assertAlmostEqual(0.1, when) + when = yield 0.1 + + loop = self.new_test_loop(gen) + + foo_running = None + + @asyncio.coroutine + def foo(): + nonlocal foo_running + foo_running = True + try: + yield from asyncio.sleep(0.2, loop=loop) + finally: + foo_running = False + return 'done' + + fut = asyncio.Task(foo(), loop=loop) + + with self.assertRaises(asyncio.TimeoutError): + loop.run_until_complete(asyncio.wait_for(fut, 0.1, loop=loop)) + self.assertTrue(fut.done()) + # it should have been cancelled due to the timeout + self.assertTrue(fut.cancelled()) + self.assertAlmostEqual(0.1, loop.time()) + self.assertEqual(foo_running, False) + + def test_wait_for_blocking(self): + loop = self.new_test_loop() + + @asyncio.coroutine + def coro(): + return 'done' + + res = loop.run_until_complete(asyncio.wait_for(coro(), + timeout=None, + loop=loop)) + self.assertEqual(res, 'done') + + def test_wait_for_with_global_loop(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.2, when) + when = yield 0 + self.assertAlmostEqual(0.01, when) + yield 0.01 + + loop = self.new_test_loop(gen) + + @asyncio.coroutine + def foo(): + yield from asyncio.sleep(0.2, loop=loop) + return 'done' + + asyncio.set_event_loop(loop) + try: + fut = asyncio.Task(foo(), loop=loop) + with self.assertRaises(asyncio.TimeoutError): + loop.run_until_complete(asyncio.wait_for(fut, 0.01)) + finally: + asyncio.set_event_loop(None) + + self.assertAlmostEqual(0.01, loop.time()) + self.assertTrue(fut.done()) + self.assertTrue(fut.cancelled()) + + def test_wait_for_race_condition(self): + + def gen(): + yield 0.1 + yield 0.1 + yield 0.1 + + loop = self.new_test_loop(gen) + + fut = asyncio.Future(loop=loop) + task = asyncio.wait_for(fut, timeout=0.2, loop=loop) + loop.call_later(0.1, fut.set_result, "ok") + res = loop.run_until_complete(task) + self.assertEqual(res, "ok") + + def test_wait(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + yield 0.15 + + loop = self.new_test_loop(gen) + + a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) + b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop) + + @asyncio.coroutine + def foo(): + done, pending = yield from asyncio.wait([b, a], loop=loop) + self.assertEqual(done, set([a, b])) + self.assertEqual(pending, set()) + return 42 + + res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + self.assertEqual(res, 42) + self.assertAlmostEqual(0.15, loop.time()) + + # Doing it again should take no time and exercise a different path. + res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + self.assertAlmostEqual(0.15, loop.time()) + self.assertEqual(res, 42) + + def test_wait_with_global_loop(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.01, when) + when = yield 0 + self.assertAlmostEqual(0.015, when) + yield 0.015 + + loop = self.new_test_loop(gen) + + a = asyncio.Task(asyncio.sleep(0.01, loop=loop), loop=loop) + b = asyncio.Task(asyncio.sleep(0.015, loop=loop), loop=loop) + + @asyncio.coroutine + def foo(): + done, pending = yield from asyncio.wait([b, a]) + self.assertEqual(done, set([a, b])) + self.assertEqual(pending, set()) + return 42 + + asyncio.set_event_loop(loop) + res = loop.run_until_complete( + asyncio.Task(foo(), loop=loop)) + + self.assertEqual(res, 42) + + def test_wait_duplicate_coroutines(self): + @asyncio.coroutine + def coro(s): + return s + c = coro('test') + + task = asyncio.Task( + asyncio.wait([c, c, coro('spam')], loop=self.loop), + loop=self.loop) + + done, pending = self.loop.run_until_complete(task) + + self.assertFalse(pending) + self.assertEqual(set(f.result() for f in done), {'test', 'spam'}) + + def test_wait_errors(self): + self.assertRaises( + ValueError, self.loop.run_until_complete, + asyncio.wait(set(), loop=self.loop)) + + # -1 is an invalid return_when value + sleep_coro = asyncio.sleep(10.0, loop=self.loop) + wait_coro = asyncio.wait([sleep_coro], return_when=-1, loop=self.loop) + self.assertRaises(ValueError, + self.loop.run_until_complete, wait_coro) + + sleep_coro.close() + + def test_wait_first_completed(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + when = yield 0 + self.assertAlmostEqual(0.1, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop) + b = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) + task = asyncio.Task( + asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED, + loop=loop), + loop=loop) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertFalse(a.done()) + self.assertTrue(b.done()) + self.assertIsNone(b.result()) + self.assertAlmostEqual(0.1, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b], loop=loop)) + + def test_wait_really_done(self): + # there is possibility that some tasks in the pending list + # became done but their callbacks haven't all been called yet + + @asyncio.coroutine + def coro1(): + yield + + @asyncio.coroutine + def coro2(): + yield + yield + + a = asyncio.Task(coro1(), loop=self.loop) + b = asyncio.Task(coro2(), loop=self.loop) + task = asyncio.Task( + asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED, + loop=self.loop), + loop=self.loop) + + done, pending = self.loop.run_until_complete(task) + self.assertEqual({a, b}, done) + self.assertTrue(a.done()) + self.assertIsNone(a.result()) + self.assertTrue(b.done()) + self.assertIsNone(b.result()) + + def test_wait_first_exception(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + # first_exception, task already has exception + a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop) + + @asyncio.coroutine + def exc(): + raise ZeroDivisionError('err') + + b = asyncio.Task(exc(), loop=loop) + task = asyncio.Task( + asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION, + loop=loop), + loop=loop) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertAlmostEqual(0, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b], loop=loop)) + + def test_wait_first_exception_in_wait(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + when = yield 0 + self.assertAlmostEqual(0.01, when) + yield 0.01 + + loop = self.new_test_loop(gen) + + # first_exception, exception during waiting + a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop) + + @asyncio.coroutine + def exc(): + yield from asyncio.sleep(0.01, loop=loop) + raise ZeroDivisionError('err') + + b = asyncio.Task(exc(), loop=loop) + task = asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION, + loop=loop) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertAlmostEqual(0.01, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b], loop=loop)) + + def test_wait_with_exception(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + yield 0.15 + + loop = self.new_test_loop(gen) + + a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) + + @asyncio.coroutine + def sleeper(): + yield from asyncio.sleep(0.15, loop=loop) + raise ZeroDivisionError('really') + + b = asyncio.Task(sleeper(), loop=loop) + + @asyncio.coroutine + def foo(): + done, pending = yield from asyncio.wait([b, a], loop=loop) + self.assertEqual(len(done), 2) + self.assertEqual(pending, set()) + errors = set(f for f in done if f.exception() is not None) + self.assertEqual(len(errors), 1) + + loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + self.assertAlmostEqual(0.15, loop.time()) + + loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + self.assertAlmostEqual(0.15, loop.time()) + + def test_wait_with_timeout(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + when = yield 0 + self.assertAlmostEqual(0.11, when) + yield 0.11 + + loop = self.new_test_loop(gen) + + a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) + b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop) + + @asyncio.coroutine + def foo(): + done, pending = yield from asyncio.wait([b, a], timeout=0.11, + loop=loop) + self.assertEqual(done, set([a])) + self.assertEqual(pending, set([b])) + + loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + self.assertAlmostEqual(0.11, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b], loop=loop)) + + def test_wait_concurrent_complete(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + when = yield 0 + self.assertAlmostEqual(0.1, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop) + b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop) + + done, pending = loop.run_until_complete( + asyncio.wait([b, a], timeout=0.1, loop=loop)) + + self.assertEqual(done, set([a])) + self.assertEqual(pending, set([b])) + self.assertAlmostEqual(0.1, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b], loop=loop)) + + def test_as_completed(self): + + def gen(): + yield 0 + yield 0 + yield 0.01 + yield 0 + + loop = self.new_test_loop(gen) + # disable "slow callback" warning + loop.slow_callback_duration = 1.0 + completed = set() + time_shifted = False + + @asyncio.coroutine + def sleeper(dt, x): + nonlocal time_shifted + yield from asyncio.sleep(dt, loop=loop) + completed.add(x) + if not time_shifted and 'a' in completed and 'b' in completed: + time_shifted = True + loop.advance_time(0.14) + return x + + a = sleeper(0.01, 'a') + b = sleeper(0.01, 'b') + c = sleeper(0.15, 'c') + + @asyncio.coroutine + def foo(): + values = [] + for f in asyncio.as_completed([b, c, a], loop=loop): + values.append((yield from f)) + return values + + res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + self.assertAlmostEqual(0.15, loop.time()) + self.assertTrue('a' in res[:2]) + self.assertTrue('b' in res[:2]) + self.assertEqual(res[2], 'c') + + # Doing it again should take no time and exercise a different path. + res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + self.assertAlmostEqual(0.15, loop.time()) + + def test_as_completed_with_timeout(self): + + def gen(): + yield + yield 0 + yield 0 + yield 0.1 + + loop = self.new_test_loop(gen) + + a = asyncio.sleep(0.1, 'a', loop=loop) + b = asyncio.sleep(0.15, 'b', loop=loop) + + @asyncio.coroutine + def foo(): + values = [] + for f in asyncio.as_completed([a, b], timeout=0.12, loop=loop): + if values: + loop.advance_time(0.02) + try: + v = yield from f + values.append((1, v)) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values + + res = loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + self.assertEqual(len(res), 2, res) + self.assertEqual(res[0], (1, 'a')) + self.assertEqual(res[1][0], 2) + self.assertIsInstance(res[1][1], asyncio.TimeoutError) + self.assertAlmostEqual(0.12, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b], loop=loop)) + + def test_as_completed_with_unused_timeout(self): + + def gen(): + yield + yield 0 + yield 0.01 + + loop = self.new_test_loop(gen) + + a = asyncio.sleep(0.01, 'a', loop=loop) + + @asyncio.coroutine + def foo(): + for f in asyncio.as_completed([a], timeout=1, loop=loop): + v = yield from f + self.assertEqual(v, 'a') + + loop.run_until_complete(asyncio.Task(foo(), loop=loop)) + + def test_as_completed_reverse_wait(self): + + def gen(): + yield 0 + yield 0.05 + yield 0 + + loop = self.new_test_loop(gen) + + a = asyncio.sleep(0.05, 'a', loop=loop) + b = asyncio.sleep(0.10, 'b', loop=loop) + fs = {a, b} + futs = list(asyncio.as_completed(fs, loop=loop)) + self.assertEqual(len(futs), 2) + + x = loop.run_until_complete(futs[1]) + self.assertEqual(x, 'a') + self.assertAlmostEqual(0.05, loop.time()) + loop.advance_time(0.05) + y = loop.run_until_complete(futs[0]) + self.assertEqual(y, 'b') + self.assertAlmostEqual(0.10, loop.time()) + + def test_as_completed_concurrent(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.05, when) + when = yield 0 + self.assertAlmostEqual(0.05, when) + yield 0.05 + + loop = self.new_test_loop(gen) + + a = asyncio.sleep(0.05, 'a', loop=loop) + b = asyncio.sleep(0.05, 'b', loop=loop) + fs = {a, b} + futs = list(asyncio.as_completed(fs, loop=loop)) + self.assertEqual(len(futs), 2) + waiter = asyncio.wait(futs, loop=loop) + done, pending = loop.run_until_complete(waiter) + self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + + def test_as_completed_duplicate_coroutines(self): + + @asyncio.coroutine + def coro(s): + return s + + @asyncio.coroutine + def runner(): + result = [] + c = coro('ham') + for f in asyncio.as_completed([c, c, coro('spam')], + loop=self.loop): + result.append((yield from f)) + return result + + fut = asyncio.Task(runner(), loop=self.loop) + self.loop.run_until_complete(fut) + result = fut.result() + self.assertEqual(set(result), {'ham', 'spam'}) + self.assertEqual(len(result), 2) + + def test_sleep(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.05, when) + when = yield 0.05 + self.assertAlmostEqual(0.1, when) + yield 0.05 + + loop = self.new_test_loop(gen) + + @asyncio.coroutine + def sleeper(dt, arg): + yield from asyncio.sleep(dt/2, loop=loop) + res = yield from asyncio.sleep(dt/2, arg, loop=loop) + return res + + t = asyncio.Task(sleeper(0.1, 'yeah'), loop=loop) + loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'yeah') + self.assertAlmostEqual(0.1, loop.time()) + + def test_sleep_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + t = asyncio.Task(asyncio.sleep(10.0, 'yeah', loop=loop), + loop=loop) + + handle = None + orig_call_later = loop.call_later + + def call_later(delay, callback, *args): + nonlocal handle + handle = orig_call_later(delay, callback, *args) + return handle + + loop.call_later = call_later + test_utils.run_briefly(loop) + + self.assertFalse(handle._cancelled) + + t.cancel() + test_utils.run_briefly(loop) + self.assertTrue(handle._cancelled) + + def test_task_cancel_sleeping_task(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(5000, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + @asyncio.coroutine + def sleep(dt): + yield from asyncio.sleep(dt, loop=loop) + + @asyncio.coroutine + def doit(): + sleeper = asyncio.Task(sleep(5000), loop=loop) + loop.call_later(0.1, sleeper.cancel) + try: + yield from sleeper + except asyncio.CancelledError: + return 'cancelled' + else: + return 'slept in' + + doer = doit() + self.assertEqual(loop.run_until_complete(doer), 'cancelled') + self.assertAlmostEqual(0.1, loop.time()) + + def test_task_cancel_waiter_future(self): + fut = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def coro(): + yield from fut + + task = asyncio.Task(coro(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertIs(task._fut_waiter, fut) + + task.cancel() + test_utils.run_briefly(self.loop) + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, task) + self.assertIsNone(task._fut_waiter) + self.assertTrue(fut.cancelled()) + + def test_step_in_completed_task(self): + @asyncio.coroutine + def notmuch(): + return 'ko' + + gen = notmuch() + task = asyncio.Task(gen, loop=self.loop) + task.set_result('ok') + + self.assertRaises(AssertionError, task._step) + gen.close() + + def test_step_result(self): + @asyncio.coroutine + def notmuch(): + yield None + yield 1 + return 'ko' + + self.assertRaises( + RuntimeError, self.loop.run_until_complete, notmuch()) + + def test_step_result_future(self): + # If coroutine returns future, task waits on this future. + + class Fut(asyncio.Future): + def __init__(self, *args, **kwds): + self.cb_added = False + super().__init__(*args, **kwds) + + def add_done_callback(self, fn): + self.cb_added = True + super().add_done_callback(fn) + + fut = Fut(loop=self.loop) + result = None + + @asyncio.coroutine + def wait_for_future(): + nonlocal result + result = yield from fut + + t = asyncio.Task(wait_for_future(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertTrue(fut.cb_added) + + res = object() + fut.set_result(res) + test_utils.run_briefly(self.loop) + self.assertIs(res, result) + self.assertTrue(t.done()) + self.assertIsNone(t.result()) + + def test_step_with_baseexception(self): + @asyncio.coroutine + def notmutch(): + raise BaseException() + + task = asyncio.Task(notmutch(), loop=self.loop) + self.assertRaises(BaseException, task._step) + + self.assertTrue(task.done()) + self.assertIsInstance(task.exception(), BaseException) + + def test_baseexception_during_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + @asyncio.coroutine + def sleeper(): + yield from asyncio.sleep(10, loop=loop) + + base_exc = BaseException() + + @asyncio.coroutine + def notmutch(): + try: + yield from sleeper() + except asyncio.CancelledError: + raise base_exc + + task = asyncio.Task(notmutch(), loop=loop) + test_utils.run_briefly(loop) + + task.cancel() + self.assertFalse(task.done()) + + self.assertRaises(BaseException, test_utils.run_briefly, loop) + + self.assertTrue(task.done()) + self.assertFalse(task.cancelled()) + self.assertIs(task.exception(), base_exc) + + def test_iscoroutinefunction(self): + def fn(): + pass + + self.assertFalse(asyncio.iscoroutinefunction(fn)) + + def fn1(): + yield + self.assertFalse(asyncio.iscoroutinefunction(fn1)) + + @asyncio.coroutine + def fn2(): + yield + self.assertTrue(asyncio.iscoroutinefunction(fn2)) + + def test_yield_vs_yield_from(self): + fut = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def wait_for_future(): + yield fut + + task = wait_for_future() + with self.assertRaises(RuntimeError): + self.loop.run_until_complete(task) + + self.assertFalse(fut.done()) + + def test_yield_vs_yield_from_generator(self): + @asyncio.coroutine + def coro(): + yield + + @asyncio.coroutine + def wait_for_future(): + gen = coro() + try: + yield gen + finally: + gen.close() + + task = wait_for_future() + self.assertRaises( + RuntimeError, + self.loop.run_until_complete, task) + + def test_coroutine_non_gen_function(self): + @asyncio.coroutine + def func(): + return 'test' + + self.assertTrue(asyncio.iscoroutinefunction(func)) + + coro = func() + self.assertTrue(asyncio.iscoroutine(coro)) + + res = self.loop.run_until_complete(coro) + self.assertEqual(res, 'test') + + def test_coroutine_non_gen_function_return_future(self): + fut = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def func(): + return fut + + @asyncio.coroutine + def coro(): + fut.set_result('test') + + t1 = asyncio.Task(func(), loop=self.loop) + t2 = asyncio.Task(coro(), loop=self.loop) + res = self.loop.run_until_complete(t1) + self.assertEqual(res, 'test') + self.assertIsNone(t2.result()) + + def test_current_task(self): + self.assertIsNone(asyncio.Task.current_task(loop=self.loop)) + + @asyncio.coroutine + def coro(loop): + self.assertTrue(asyncio.Task.current_task(loop=loop) is task) + + task = asyncio.Task(coro(self.loop), loop=self.loop) + self.loop.run_until_complete(task) + self.assertIsNone(asyncio.Task.current_task(loop=self.loop)) + + def test_current_task_with_interleaving_tasks(self): + self.assertIsNone(asyncio.Task.current_task(loop=self.loop)) + + fut1 = asyncio.Future(loop=self.loop) + fut2 = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def coro1(loop): + self.assertTrue(asyncio.Task.current_task(loop=loop) is task1) + yield from fut1 + self.assertTrue(asyncio.Task.current_task(loop=loop) is task1) + fut2.set_result(True) + + @asyncio.coroutine + def coro2(loop): + self.assertTrue(asyncio.Task.current_task(loop=loop) is task2) + fut1.set_result(True) + yield from fut2 + self.assertTrue(asyncio.Task.current_task(loop=loop) is task2) + + task1 = asyncio.Task(coro1(self.loop), loop=self.loop) + task2 = asyncio.Task(coro2(self.loop), loop=self.loop) + + self.loop.run_until_complete(asyncio.wait((task1, task2), + loop=self.loop)) + self.assertIsNone(asyncio.Task.current_task(loop=self.loop)) + + # Some thorough tests for cancellation propagation through + # coroutines, tasks and wait(). + + def test_yield_future_passes_cancel(self): + # Cancelling outer() cancels inner() cancels waiter. + proof = 0 + waiter = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def inner(): + nonlocal proof + try: + yield from waiter + except asyncio.CancelledError: + proof += 1 + raise + else: + self.fail('got past sleep() in inner()') + + @asyncio.coroutine + def outer(): + nonlocal proof + try: + yield from inner() + except asyncio.CancelledError: + proof += 100 # Expect this path. + else: + proof += 10 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + self.loop.run_until_complete(f) + self.assertEqual(proof, 101) + self.assertTrue(waiter.cancelled()) + + def test_yield_wait_does_not_shield_cancel(self): + # Cancelling outer() makes wait() return early, leaves inner() + # running. + proof = 0 + waiter = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def inner(): + nonlocal proof + yield from waiter + proof += 1 + + @asyncio.coroutine + def outer(): + nonlocal proof + d, p = yield from asyncio.wait([inner()], loop=self.loop) + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, f) + waiter.set_result(None) + test_utils.run_briefly(self.loop) + self.assertEqual(proof, 1) + + def test_shield_result(self): + inner = asyncio.Future(loop=self.loop) + outer = asyncio.shield(inner) + inner.set_result(42) + res = self.loop.run_until_complete(outer) + self.assertEqual(res, 42) + + def test_shield_exception(self): + inner = asyncio.Future(loop=self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + exc = RuntimeError('expected') + inner.set_exception(exc) + test_utils.run_briefly(self.loop) + self.assertIs(outer.exception(), exc) + + def test_shield_cancel(self): + inner = asyncio.Future(loop=self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + inner.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(outer.cancelled()) + + def test_shield_shortcut(self): + fut = asyncio.Future(loop=self.loop) + fut.set_result(42) + res = self.loop.run_until_complete(asyncio.shield(fut)) + self.assertEqual(res, 42) + + def test_shield_effect(self): + # Cancelling outer() does not affect inner(). + proof = 0 + waiter = asyncio.Future(loop=self.loop) + + @asyncio.coroutine + def inner(): + nonlocal proof + yield from waiter + proof += 1 + + @asyncio.coroutine + def outer(): + nonlocal proof + yield from asyncio.shield(inner(), loop=self.loop) + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) + waiter.set_result(None) + test_utils.run_briefly(self.loop) + self.assertEqual(proof, 1) + + def test_shield_gather(self): + child1 = asyncio.Future(loop=self.loop) + child2 = asyncio.Future(loop=self.loop) + parent = asyncio.gather(child1, child2, loop=self.loop) + outer = asyncio.shield(parent, loop=self.loop) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(outer.cancelled()) + child1.set_result(1) + child2.set_result(2) + test_utils.run_briefly(self.loop) + self.assertEqual(parent.result(), [1, 2]) + + def test_gather_shield(self): + child1 = asyncio.Future(loop=self.loop) + child2 = asyncio.Future(loop=self.loop) + inner1 = asyncio.shield(child1, loop=self.loop) + inner2 = asyncio.shield(child2, loop=self.loop) + parent = asyncio.gather(inner1, inner2, loop=self.loop) + test_utils.run_briefly(self.loop) + parent.cancel() + # This should cancel inner1 and inner2 but bot child1 and child2. + test_utils.run_briefly(self.loop) + self.assertIsInstance(parent.exception(), asyncio.CancelledError) + self.assertTrue(inner1.cancelled()) + self.assertTrue(inner2.cancelled()) + child1.set_result(1) + child2.set_result(2) + test_utils.run_briefly(self.loop) + + def test_as_completed_invalid_args(self): + fut = asyncio.Future(loop=self.loop) + + # as_completed() expects a list of futures, not a future instance + self.assertRaises(TypeError, self.loop.run_until_complete, + asyncio.as_completed(fut, loop=self.loop)) + coro = coroutine_function() + self.assertRaises(TypeError, self.loop.run_until_complete, + asyncio.as_completed(coro, loop=self.loop)) + coro.close() + + def test_wait_invalid_args(self): + fut = asyncio.Future(loop=self.loop) + + # wait() expects a list of futures, not a future instance + self.assertRaises(TypeError, self.loop.run_until_complete, + asyncio.wait(fut, loop=self.loop)) + coro = coroutine_function() + self.assertRaises(TypeError, self.loop.run_until_complete, + asyncio.wait(coro, loop=self.loop)) + coro.close() + + # wait() expects at least a future + self.assertRaises(ValueError, self.loop.run_until_complete, + asyncio.wait([], loop=self.loop)) + + def test_corowrapper_mocks_generator(self): + + def check(): + # A function that asserts various things. + # Called twice, with different debug flag values. + + @asyncio.coroutine + def coro(): + # The actual coroutine. + self.assertTrue(gen.gi_running) + yield from fut + + # A completed Future used to run the coroutine. + fut = asyncio.Future(loop=self.loop) + fut.set_result(None) + + # Call the coroutine. + gen = coro() + + # Check some properties. + self.assertTrue(asyncio.iscoroutine(gen)) + self.assertIsInstance(gen.gi_frame, types.FrameType) + self.assertFalse(gen.gi_running) + self.assertIsInstance(gen.gi_code, types.CodeType) + + # Run it. + self.loop.run_until_complete(gen) + + # The frame should have changed. + self.assertIsNone(gen.gi_frame) + + # Test with debug flag cleared. + with set_coroutine_debug(False): + check() + + # Test with debug flag set. + with set_coroutine_debug(True): + check() + + def test_yield_from_corowrapper(self): + with set_coroutine_debug(True): + @asyncio.coroutine + def t1(): + return (yield from t2()) + + @asyncio.coroutine + def t2(): + f = asyncio.Future(loop=self.loop) + asyncio.Task(t3(f), loop=self.loop) + return (yield from f) + + @asyncio.coroutine + def t3(f): + f.set_result((1, 2, 3)) + + task = asyncio.Task(t1(), loop=self.loop) + val = self.loop.run_until_complete(task) + self.assertEqual(val, (1, 2, 3)) + + def test_yield_from_corowrapper_send(self): + def foo(): + a = yield + return a + + def call(arg): + cw = asyncio.coroutines.CoroWrapper(foo()) + cw.send(None) + try: + cw.send(arg) + except StopIteration as ex: + return ex.args[0] + else: + raise AssertionError('StopIteration was expected') + + self.assertEqual(call((1, 2)), (1, 2)) + self.assertEqual(call('spam'), 'spam') + + def test_corowrapper_weakref(self): + wd = weakref.WeakValueDictionary() + def foo(): yield from [] + cw = asyncio.coroutines.CoroWrapper(foo()) + wd['cw'] = cw # Would fail without __weakref__ slot. + cw.gen = None # Suppress warning from __del__. + + def test_corowrapper_throw(self): + # Issue 429: CoroWrapper.throw must be compatible with gen.throw + def foo(): + value = None + while True: + try: + value = yield value + except Exception as e: + value = e + + exception = Exception("foo") + cw = asyncio.coroutines.CoroWrapper(foo()) + cw.send(None) + self.assertIs(exception, cw.throw(exception)) + + cw = asyncio.coroutines.CoroWrapper(foo()) + cw.send(None) + self.assertIs(exception, cw.throw(Exception, exception)) + + cw = asyncio.coroutines.CoroWrapper(foo()) + cw.send(None) + exception = cw.throw(Exception, "foo") + self.assertIsInstance(exception, Exception) + self.assertEqual(exception.args, ("foo", )) + + cw = asyncio.coroutines.CoroWrapper(foo()) + cw.send(None) + exception = cw.throw(Exception, "foo", None) + self.assertIsInstance(exception, Exception) + self.assertEqual(exception.args, ("foo", )) + + @unittest.skipUnless(PY34, + 'need python 3.4 or later') + def test_log_destroyed_pending_task(self): + @asyncio.coroutine + def kill_me(loop): + future = asyncio.Future(loop=loop) + yield from future + # at this point, the only reference to kill_me() task is + # the Task._wakeup() method in future._callbacks + raise Exception("code never reached") + + mock_handler = mock.Mock() + self.loop.set_debug(True) + self.loop.set_exception_handler(mock_handler) + + # schedule the task + coro = kill_me(self.loop) + task = asyncio.ensure_future(coro, loop=self.loop) + self.assertEqual(asyncio.Task.all_tasks(loop=self.loop), {task}) + + # execute the task so it waits for future + self.loop._run_once() + self.assertEqual(len(self.loop._ready), 0) + + # remove the future used in kill_me(), and references to the task + del coro.gi_frame.f_locals['future'] + coro = None + source_traceback = task._source_traceback + task = None + + # no more reference to kill_me() task: the task is destroyed by the GC + support.gc_collect() + + self.assertEqual(asyncio.Task.all_tasks(loop=self.loop), set()) + + mock_handler.assert_called_with(self.loop, { + 'message': 'Task was destroyed but it is pending!', + 'task': mock.ANY, + 'source_traceback': source_traceback, + }) + mock_handler.reset_mock() + + @mock.patch('asyncio.coroutines.logger') + def test_coroutine_never_yielded(self, m_log): + with set_coroutine_debug(True): + @asyncio.coroutine + def coro_noop(): + pass + + tb_filename = __file__ + tb_lineno = sys._getframe().f_lineno + 2 + # create a coroutine object but don't use it + coro_noop() + support.gc_collect() + + self.assertTrue(m_log.error.called) + message = m_log.error.call_args[0][0] + func_filename, func_lineno = test_utils.get_function_source(coro_noop) + + regex = (r'^ ' + r'was never yielded from\n' + r'Coroutine object created at \(most recent call last\):\n' + r'.*\n' + r' File "%s", line %s, in test_coroutine_never_yielded\n' + r' coro_noop\(\)$' + % (re.escape(coro_noop.__qualname__), + re.escape(func_filename), func_lineno, + re.escape(tb_filename), tb_lineno)) + + self.assertRegex(message, re.compile(regex, re.DOTALL)) + + def test_return_coroutine_from_coroutine(self): + """Return of @asyncio.coroutine()-wrapped function generator object + from @asyncio.coroutine()-wrapped function should have same effect as + returning generator object or Future.""" + def check(): + @asyncio.coroutine + def outer_coro(): + @asyncio.coroutine + def inner_coro(): + return 1 + + return inner_coro() + + result = self.loop.run_until_complete(outer_coro()) + self.assertEqual(result, 1) + + # Test with debug flag cleared. + with set_coroutine_debug(False): + check() + + # Test with debug flag set. + with set_coroutine_debug(True): + check() + + def test_task_source_traceback(self): + self.loop.set_debug(True) + + task = asyncio.Task(coroutine_function(), loop=self.loop) + lineno = sys._getframe().f_lineno - 1 + self.assertIsInstance(task._source_traceback, list) + self.assertEqual(task._source_traceback[-1][:3], + (__file__, + lineno, + 'test_task_source_traceback')) + self.loop.run_until_complete(task) + + def _test_cancel_wait_for(self, timeout): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + @asyncio.coroutine + def blocking_coroutine(): + fut = asyncio.Future(loop=loop) + # Block: fut result is never set + yield from fut + + task = loop.create_task(blocking_coroutine()) + + wait = loop.create_task(asyncio.wait_for(task, timeout, loop=loop)) + loop.call_soon(wait.cancel) + + self.assertRaises(asyncio.CancelledError, + loop.run_until_complete, wait) + + # Python issue #23219: cancelling the wait must also cancel the task + self.assertTrue(task.cancelled()) + + def test_cancel_blocking_wait_for(self): + self._test_cancel_wait_for(None) + + def test_cancel_wait_for(self): + self._test_cancel_wait_for(60.0) + + def test_cancel_gather(self): + """Ensure that a gathering future refuses to be cancelled once all + children are done""" + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + fut = asyncio.Future(loop=loop) + # The indirection fut->child_coro is needed since otherwise the + # gathering task is done at the same time as the child future + def child_coro(): + return (yield from fut) + gather_future = asyncio.gather(child_coro(), loop=loop) + gather_task = asyncio.ensure_future(gather_future, loop=loop) + + cancel_result = None + def cancelling_callback(_): + nonlocal cancel_result + cancel_result = gather_task.cancel() + fut.add_done_callback(cancelling_callback) + + fut.set_result(42) # calls the cancelling_callback after fut is done() + + # At this point the task should complete. + loop.run_until_complete(gather_task) + + # Python issue #26923: asyncio.gather drops cancellation + self.assertEqual(cancel_result, False) + self.assertFalse(gather_task.cancelled()) + self.assertEqual(gather_task.result(), [42]) + + +class GatherTestsBase: + + def setUp(self): + super().setUp() + self.one_loop = self.new_test_loop() + self.other_loop = self.new_test_loop() + self.set_event_loop(self.one_loop, cleanup=False) + + def _run_loop(self, loop): + while loop._ready: + test_utils.run_briefly(loop) + + def _check_success(self, **kwargs): + a, b, c = [asyncio.Future(loop=self.one_loop) for i in range(3)] + fut = asyncio.gather(*self.wrap_futures(a, b, c), **kwargs) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + b.set_result(1) + a.set_result(2) + self._run_loop(self.one_loop) + self.assertEqual(cb.called, False) + self.assertFalse(fut.done()) + c.set_result(3) + self._run_loop(self.one_loop) + cb.assert_called_once_with(fut) + self.assertEqual(fut.result(), [2, 1, 3]) + + def test_success(self): + self._check_success() + self._check_success(return_exceptions=False) + + def test_result_exception_success(self): + self._check_success(return_exceptions=True) + + def test_one_exception(self): + a, b, c, d, e = [asyncio.Future(loop=self.one_loop) for i in range(5)] + fut = asyncio.gather(*self.wrap_futures(a, b, c, d, e)) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + exc = ZeroDivisionError() + a.set_result(1) + b.set_exception(exc) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertIs(fut.exception(), exc) + # Does nothing + c.set_result(3) + d.cancel() + e.set_exception(RuntimeError()) + e.exception() + + def test_return_exceptions(self): + a, b, c, d = [asyncio.Future(loop=self.one_loop) for i in range(4)] + fut = asyncio.gather(*self.wrap_futures(a, b, c, d), + return_exceptions=True) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + exc = ZeroDivisionError() + exc2 = RuntimeError() + b.set_result(1) + c.set_exception(exc) + a.set_result(3) + self._run_loop(self.one_loop) + self.assertFalse(fut.done()) + d.set_exception(exc2) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertEqual(fut.result(), [3, 1, exc, exc2]) + + def test_env_var_debug(self): + aio_path = os.path.dirname(os.path.dirname(asyncio.__file__)) + + code = '\n'.join(( + 'import asyncio.coroutines', + 'print(asyncio.coroutines._DEBUG)')) + + # Test with -E to not fail if the unit test was run with + # PYTHONASYNCIODEBUG set to a non-empty string + sts, stdout, stderr = assert_python_ok('-E', '-c', code, + PYTHONPATH=aio_path) + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='', + PYTHONPATH=aio_path) + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='1', + PYTHONPATH=aio_path) + self.assertEqual(stdout.rstrip(), b'True') + + sts, stdout, stderr = assert_python_ok('-E', '-c', code, + PYTHONASYNCIODEBUG='1', + PYTHONPATH=aio_path) + self.assertEqual(stdout.rstrip(), b'False') + + +class FutureGatherTests(GatherTestsBase, test_utils.TestCase): + + def wrap_futures(self, *futures): + return futures + + def _check_empty_sequence(self, seq_or_iter): + asyncio.set_event_loop(self.one_loop) + self.addCleanup(asyncio.set_event_loop, None) + fut = asyncio.gather(*seq_or_iter) + self.assertIsInstance(fut, asyncio.Future) + self.assertIs(fut._loop, self.one_loop) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + self.assertEqual(fut.result(), []) + fut = asyncio.gather(*seq_or_iter, loop=self.other_loop) + self.assertIs(fut._loop, self.other_loop) + + def test_constructor_empty_sequence(self): + self._check_empty_sequence([]) + self._check_empty_sequence(()) + self._check_empty_sequence(set()) + self._check_empty_sequence(iter("")) + + def test_constructor_heterogenous_futures(self): + fut1 = asyncio.Future(loop=self.one_loop) + fut2 = asyncio.Future(loop=self.other_loop) + with self.assertRaises(ValueError): + asyncio.gather(fut1, fut2) + with self.assertRaises(ValueError): + asyncio.gather(fut1, loop=self.other_loop) + + def test_constructor_homogenous_futures(self): + children = [asyncio.Future(loop=self.other_loop) for i in range(3)] + fut = asyncio.gather(*children) + self.assertIs(fut._loop, self.other_loop) + self._run_loop(self.other_loop) + self.assertFalse(fut.done()) + fut = asyncio.gather(*children, loop=self.other_loop) + self.assertIs(fut._loop, self.other_loop) + self._run_loop(self.other_loop) + self.assertFalse(fut.done()) + + def test_one_cancellation(self): + a, b, c, d, e = [asyncio.Future(loop=self.one_loop) for i in range(5)] + fut = asyncio.gather(a, b, c, d, e) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + a.set_result(1) + b.cancel() + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertFalse(fut.cancelled()) + self.assertIsInstance(fut.exception(), asyncio.CancelledError) + # Does nothing + c.set_result(3) + d.cancel() + e.set_exception(RuntimeError()) + e.exception() + + def test_result_exception_one_cancellation(self): + a, b, c, d, e, f = [asyncio.Future(loop=self.one_loop) + for i in range(6)] + fut = asyncio.gather(a, b, c, d, e, f, return_exceptions=True) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + a.set_result(1) + zde = ZeroDivisionError() + b.set_exception(zde) + c.cancel() + self._run_loop(self.one_loop) + self.assertFalse(fut.done()) + d.set_result(3) + e.cancel() + rte = RuntimeError() + f.set_exception(rte) + res = self.one_loop.run_until_complete(fut) + self.assertIsInstance(res[2], asyncio.CancelledError) + self.assertIsInstance(res[4], asyncio.CancelledError) + res[2] = res[4] = None + self.assertEqual(res, [1, zde, None, 3, None, rte]) + cb.assert_called_once_with(fut) + + +class CoroutineGatherTests(GatherTestsBase, test_utils.TestCase): + + def setUp(self): + super().setUp() + asyncio.set_event_loop(self.one_loop) + + def wrap_futures(self, *futures): + coros = [] + for fut in futures: + @asyncio.coroutine + def coro(fut=fut): + return (yield from fut) + coros.append(coro()) + return coros + + def test_constructor_loop_selection(self): + @asyncio.coroutine + def coro(): + return 'abc' + gen1 = coro() + gen2 = coro() + fut = asyncio.gather(gen1, gen2) + self.assertIs(fut._loop, self.one_loop) + self.one_loop.run_until_complete(fut) + + self.set_event_loop(self.other_loop, cleanup=False) + gen3 = coro() + gen4 = coro() + fut2 = asyncio.gather(gen3, gen4, loop=self.other_loop) + self.assertIs(fut2._loop, self.other_loop) + self.other_loop.run_until_complete(fut2) + + def test_duplicate_coroutines(self): + @asyncio.coroutine + def coro(s): + return s + c = coro('abc') + fut = asyncio.gather(c, c, coro('def'), c, loop=self.one_loop) + self._run_loop(self.one_loop) + self.assertEqual(fut.result(), ['abc', 'abc', 'def', 'abc']) + + def test_cancellation_broadcast(self): + # Cancelling outer() cancels all children. + proof = 0 + waiter = asyncio.Future(loop=self.one_loop) + + @asyncio.coroutine + def inner(): + nonlocal proof + yield from waiter + proof += 1 + + child1 = asyncio.ensure_future(inner(), loop=self.one_loop) + child2 = asyncio.ensure_future(inner(), loop=self.one_loop) + gatherer = None + + @asyncio.coroutine + def outer(): + nonlocal proof, gatherer + gatherer = asyncio.gather(child1, child2, loop=self.one_loop) + yield from gatherer + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.one_loop) + test_utils.run_briefly(self.one_loop) + self.assertTrue(f.cancel()) + with self.assertRaises(asyncio.CancelledError): + self.one_loop.run_until_complete(f) + self.assertFalse(gatherer.cancel()) + self.assertTrue(waiter.cancelled()) + self.assertTrue(child1.cancelled()) + self.assertTrue(child2.cancelled()) + test_utils.run_briefly(self.one_loop) + self.assertEqual(proof, 0) + + def test_exception_marking(self): + # Test for the first line marked "Mark exception retrieved." + + @asyncio.coroutine + def inner(f): + yield from f + raise RuntimeError('should not be ignored') + + a = asyncio.Future(loop=self.one_loop) + b = asyncio.Future(loop=self.one_loop) + + @asyncio.coroutine + def outer(): + yield from asyncio.gather(inner(a), inner(b), loop=self.one_loop) + + f = asyncio.ensure_future(outer(), loop=self.one_loop) + test_utils.run_briefly(self.one_loop) + a.set_result(None) + test_utils.run_briefly(self.one_loop) + b.set_result(None) + test_utils.run_briefly(self.one_loop) + self.assertIsInstance(f.exception(), RuntimeError) + + +class RunCoroutineThreadsafeTests(test_utils.TestCase): + """Test case for asyncio.run_coroutine_threadsafe.""" + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) # Will cleanup properly + + @asyncio.coroutine + def add(self, a, b, fail=False, cancel=False): + """Wait 0.05 second and return a + b.""" + yield from asyncio.sleep(0.05, loop=self.loop) + if fail: + raise RuntimeError("Fail!") + if cancel: + asyncio.tasks.Task.current_task(self.loop).cancel() + yield + return a + b + + def target(self, fail=False, cancel=False, timeout=None, + advance_coro=False): + """Run add coroutine in the event loop.""" + coro = self.add(1, 2, fail=fail, cancel=cancel) + future = asyncio.run_coroutine_threadsafe(coro, self.loop) + if advance_coro: + # this is for test_run_coroutine_threadsafe_task_factory_exception; + # otherwise it spills errors and breaks **other** unittests, since + # 'target' is interacting with threads. + + # With this call, `coro` will be advanced, so that + # CoroWrapper.__del__ won't do anything when asyncio tests run + # in debug mode. + self.loop.call_soon_threadsafe(coro.send, None) + try: + return future.result(timeout) + finally: + future.done() or future.cancel() + + def test_run_coroutine_threadsafe(self): + """Test coroutine submission from a thread to an event loop.""" + future = self.loop.run_in_executor(None, self.target) + result = self.loop.run_until_complete(future) + self.assertEqual(result, 3) + + def test_run_coroutine_threadsafe_with_exception(self): + """Test coroutine submission from a thread to an event loop + when an exception is raised.""" + future = self.loop.run_in_executor(None, self.target, True) + with self.assertRaises(RuntimeError) as exc_context: + self.loop.run_until_complete(future) + self.assertIn("Fail!", exc_context.exception.args) + + def test_run_coroutine_threadsafe_with_timeout(self): + """Test coroutine submission from a thread to an event loop + when a timeout is raised.""" + callback = lambda: self.target(timeout=0) + future = self.loop.run_in_executor(None, callback) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(future) + test_utils.run_briefly(self.loop) + # Check that there's no pending task (add has been cancelled) + for task in asyncio.Task.all_tasks(self.loop): + self.assertTrue(task.done()) + + def test_run_coroutine_threadsafe_task_cancelled(self): + """Test coroutine submission from a tread to an event loop + when the task is cancelled.""" + callback = lambda: self.target(cancel=True) + future = self.loop.run_in_executor(None, callback) + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(future) + + def test_run_coroutine_threadsafe_task_factory_exception(self): + """Test coroutine submission from a tread to an event loop + when the task factory raise an exception.""" + # Schedule the target + future = self.loop.run_in_executor( + None, lambda: self.target(advance_coro=True)) + # Set corrupted task factory + self.loop.set_task_factory(lambda loop, coro: wrong_name) + # Set exception handler + callback = test_utils.MockCallback() + self.loop.set_exception_handler(callback) + # Run event loop + with self.assertRaises(NameError) as exc_context: + self.loop.run_until_complete(future) + # Check exceptions + self.assertIn('wrong_name', exc_context.exception.args[0]) + self.assertEqual(len(callback.call_args_list), 1) + (loop, context), kwargs = callback.call_args + self.assertEqual(context['exception'], exc_context.exception) + + +class SleepTests(test_utils.TestCase): + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(None) + + def tearDown(self): + self.loop.close() + self.loop = None + super().tearDown() + + def test_sleep_zero(self): + result = 0 + + def inc_result(num): + nonlocal result + result += num + + @asyncio.coroutine + def coro(): + self.loop.call_soon(inc_result, 1) + self.assertEqual(result, 0) + num = yield from asyncio.sleep(0, loop=self.loop, result=10) + self.assertEqual(result, 1) # inc'ed by call_soon + inc_result(num) # num should be 11 + + self.loop.run_until_complete(coro()) + self.assertEqual(result, 11) + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_transports.py b/thirdparty/asyncio/tests/test_transports.py new file mode 100644 index 0000000..3b6e3d6 --- /dev/null +++ b/thirdparty/asyncio/tests/test_transports.py @@ -0,0 +1,91 @@ +"""Tests for transports.py.""" + +import unittest +from unittest import mock + +import asyncio +from asyncio import transports + + +class TransportTests(unittest.TestCase): + + def test_ctor_extra_is_none(self): + transport = asyncio.Transport() + self.assertEqual(transport._extra, {}) + + def test_get_extra_info(self): + transport = asyncio.Transport({'extra': 'info'}) + self.assertEqual('info', transport.get_extra_info('extra')) + self.assertIsNone(transport.get_extra_info('unknown')) + + default = object() + self.assertIs(default, transport.get_extra_info('unknown', default)) + + def test_writelines(self): + transport = asyncio.Transport() + transport.write = mock.Mock() + + transport.writelines([b'line1', + bytearray(b'line2'), + memoryview(b'line3')]) + self.assertEqual(1, transport.write.call_count) + transport.write.assert_called_with(b'line1line2line3') + + def test_not_implemented(self): + transport = asyncio.Transport() + + self.assertRaises(NotImplementedError, + transport.set_write_buffer_limits) + self.assertRaises(NotImplementedError, transport.get_write_buffer_size) + self.assertRaises(NotImplementedError, transport.write, 'data') + self.assertRaises(NotImplementedError, transport.write_eof) + self.assertRaises(NotImplementedError, transport.can_write_eof) + self.assertRaises(NotImplementedError, transport.pause_reading) + self.assertRaises(NotImplementedError, transport.resume_reading) + self.assertRaises(NotImplementedError, transport.close) + self.assertRaises(NotImplementedError, transport.abort) + + def test_dgram_not_implemented(self): + transport = asyncio.DatagramTransport() + + self.assertRaises(NotImplementedError, transport.sendto, 'data') + self.assertRaises(NotImplementedError, transport.abort) + + def test_subprocess_transport_not_implemented(self): + transport = asyncio.SubprocessTransport() + + self.assertRaises(NotImplementedError, transport.get_pid) + self.assertRaises(NotImplementedError, transport.get_returncode) + self.assertRaises(NotImplementedError, transport.get_pipe_transport, 1) + self.assertRaises(NotImplementedError, transport.send_signal, 1) + self.assertRaises(NotImplementedError, transport.terminate) + self.assertRaises(NotImplementedError, transport.kill) + + def test_flowcontrol_mixin_set_write_limits(self): + + class MyTransport(transports._FlowControlMixin, + transports.Transport): + + def get_write_buffer_size(self): + return 512 + + loop = mock.Mock() + transport = MyTransport(loop=loop) + transport._protocol = mock.Mock() + + self.assertFalse(transport._protocol_paused) + + with self.assertRaisesRegex(ValueError, 'high.*must be >= low'): + transport.set_write_buffer_limits(high=0, low=1) + + transport.set_write_buffer_limits(high=1024, low=128) + self.assertFalse(transport._protocol_paused) + self.assertEqual(transport.get_write_buffer_limits(), (128, 1024)) + + transport.set_write_buffer_limits(high=256, low=128) + self.assertTrue(transport._protocol_paused) + self.assertEqual(transport.get_write_buffer_limits(), (128, 256)) + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_unix_events.py b/thirdparty/asyncio/tests/test_unix_events.py new file mode 100644 index 0000000..83a035e --- /dev/null +++ b/thirdparty/asyncio/tests/test_unix_events.py @@ -0,0 +1,1584 @@ +"""Tests for unix_events.py.""" + +import collections +import errno +import io +import os +import signal +import socket +import stat +import sys +import tempfile +import threading +import unittest +import warnings +from unittest import mock + +if sys.platform == 'win32': + raise unittest.SkipTest('UNIX only') + + +import asyncio +from asyncio import log +from asyncio import test_utils +from asyncio import unix_events + + +MOCK_ANY = mock.ANY + + +def close_pipe_transport(transport): + # Don't call transport.close() because the event loop and the selector + # are mocked + if transport._pipe is None: + return + transport._pipe.close() + transport._pipe = None + + +@unittest.skipUnless(signal, 'Signals are not supported') +class SelectorEventLoopSignalTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.SelectorEventLoop() + self.set_event_loop(self.loop) + + def test_check_signal(self): + self.assertRaises( + TypeError, self.loop._check_signal, '1') + self.assertRaises( + ValueError, self.loop._check_signal, signal.NSIG + 1) + + def test_handle_signal_no_handler(self): + self.loop._handle_signal(signal.NSIG + 1) + + def test_handle_signal_cancelled_handler(self): + h = asyncio.Handle(mock.Mock(), (), + loop=mock.Mock()) + h.cancel() + self.loop._signal_handlers[signal.NSIG + 1] = h + self.loop.remove_signal_handler = mock.Mock() + self.loop._handle_signal(signal.NSIG + 1) + self.loop.remove_signal_handler.assert_called_with(signal.NSIG + 1) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_setup_error(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.set_wakeup_fd.side_effect = ValueError + + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_coroutine_error(self, m_signal): + m_signal.NSIG = signal.NSIG + + @asyncio.coroutine + def simple_coroutine(): + yield from [] + + # callback must not be a coroutine function + coro_func = simple_coroutine + coro_obj = coro_func() + self.addCleanup(coro_obj.close) + for func in (coro_func, coro_obj): + self.assertRaisesRegex( + TypeError, 'coroutines cannot be used with add_signal_handler', + self.loop.add_signal_handler, + signal.SIGINT, func) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler(self, m_signal): + m_signal.NSIG = signal.NSIG + + cb = lambda: True + self.loop.add_signal_handler(signal.SIGHUP, cb) + h = self.loop._signal_handlers.get(signal.SIGHUP) + self.assertIsInstance(h, asyncio.Handle) + self.assertEqual(h._callback, cb) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_install_error(self, m_signal): + m_signal.NSIG = signal.NSIG + + def set_wakeup_fd(fd): + if fd == -1: + raise ValueError() + m_signal.set_wakeup_fd = set_wakeup_fd + + class Err(OSError): + errno = errno.EFAULT + m_signal.signal.side_effect = Err + + self.assertRaises( + Err, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_add_signal_handler_install_error2(self, m_logging, m_signal): + m_signal.NSIG = signal.NSIG + + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + + self.loop._signal_handlers[signal.SIGHUP] = lambda: True + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + self.assertFalse(m_logging.info.called) + self.assertEqual(1, m_signal.set_wakeup_fd.call_count) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_add_signal_handler_install_error3(self, m_logging, m_signal): + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + m_signal.NSIG = signal.NSIG + + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + self.assertFalse(m_logging.info.called) + self.assertEqual(2, m_signal.set_wakeup_fd.call_count) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler(self, m_signal): + m_signal.NSIG = signal.NSIG + + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + self.assertTrue( + self.loop.remove_signal_handler(signal.SIGHUP)) + self.assertTrue(m_signal.set_wakeup_fd.called) + self.assertTrue(m_signal.signal.called) + self.assertEqual( + (signal.SIGHUP, m_signal.SIG_DFL), m_signal.signal.call_args[0]) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_2(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.SIGINT = signal.SIGINT + + self.loop.add_signal_handler(signal.SIGINT, lambda: True) + self.loop._signal_handlers[signal.SIGHUP] = object() + m_signal.set_wakeup_fd.reset_mock() + + self.assertTrue( + self.loop.remove_signal_handler(signal.SIGINT)) + self.assertFalse(m_signal.set_wakeup_fd.called) + self.assertTrue(m_signal.signal.called) + self.assertEqual( + (signal.SIGINT, m_signal.default_int_handler), + m_signal.signal.call_args[0]) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_remove_signal_handler_cleanup_error(self, m_logging, m_signal): + m_signal.NSIG = signal.NSIG + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + m_signal.set_wakeup_fd.side_effect = ValueError + + self.loop.remove_signal_handler(signal.SIGHUP) + self.assertTrue(m_logging.info) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_error(self, m_signal): + m_signal.NSIG = signal.NSIG + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + m_signal.signal.side_effect = OSError + + self.assertRaises( + OSError, self.loop.remove_signal_handler, signal.SIGHUP) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_error2(self, m_signal): + m_signal.NSIG = signal.NSIG + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + + self.assertRaises( + RuntimeError, self.loop.remove_signal_handler, signal.SIGHUP) + + @mock.patch('asyncio.unix_events.signal') + def test_close(self, m_signal): + m_signal.NSIG = signal.NSIG + + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + self.loop.add_signal_handler(signal.SIGCHLD, lambda: True) + + self.assertEqual(len(self.loop._signal_handlers), 2) + + m_signal.set_wakeup_fd.reset_mock() + + self.loop.close() + + self.assertEqual(len(self.loop._signal_handlers), 0) + m_signal.set_wakeup_fd.assert_called_once_with(-1) + + +@unittest.skipUnless(hasattr(socket, 'AF_UNIX'), + 'UNIX Sockets are not supported') +class SelectorEventLoopUnixSocketTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.SelectorEventLoop() + self.set_event_loop(self.loop) + + def test_create_unix_server_existing_path_sock(self): + with test_utils.unix_socket_path() as path: + sock = socket.socket(socket.AF_UNIX) + sock.bind(path) + sock.listen(1) + sock.close() + + coro = self.loop.create_unix_server(lambda: None, path) + srv = self.loop.run_until_complete(coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + def test_create_unix_server_existing_path_nonsock(self): + with tempfile.NamedTemporaryFile() as file: + coro = self.loop.create_unix_server(lambda: None, file.name) + with self.assertRaisesRegex(OSError, + 'Address.*is already in use'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_ssl_bool(self): + coro = self.loop.create_unix_server(lambda: None, path='spam', + ssl=True) + with self.assertRaisesRegex(TypeError, + 'ssl argument must be an SSLContext'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_nopath_nosock(self): + coro = self.loop.create_unix_server(lambda: None, path=None) + with self.assertRaisesRegex(ValueError, + 'path was not specified, and no sock'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_path_inetsock(self): + sock = socket.socket() + with sock: + coro = self.loop.create_unix_server(lambda: None, path=None, + sock=sock) + with self.assertRaisesRegex(ValueError, + 'A UNIX Domain Stream.*was expected'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_path_inetsock(self): + sock = socket.socket() + with sock: + coro = self.loop.create_unix_connection(lambda: None, path=None, + sock=sock) + with self.assertRaisesRegex(ValueError, + 'A UNIX Domain Stream.*was expected'): + self.loop.run_until_complete(coro) + + @mock.patch('asyncio.unix_events.socket') + def test_create_unix_server_bind_error(self, m_socket): + # Ensure that the socket is closed on any bind error + sock = mock.Mock() + m_socket.socket.return_value = sock + + sock.bind.side_effect = OSError + coro = self.loop.create_unix_server(lambda: None, path="/test") + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + sock.bind.side_effect = MemoryError + coro = self.loop.create_unix_server(lambda: None, path="/test") + with self.assertRaises(MemoryError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + def test_create_unix_connection_path_sock(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, sock=object()) + with self.assertRaisesRegex(ValueError, 'path and sock can not be'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_nopath_nosock(self): + coro = self.loop.create_unix_connection( + lambda: None, None) + with self.assertRaisesRegex(ValueError, + 'no path and sock were specified'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_nossl_serverhost(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, server_hostname='spam') + with self.assertRaisesRegex(ValueError, + 'server_hostname is only meaningful'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_ssl_noserverhost(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, ssl=True) + + with self.assertRaisesRegex( + ValueError, 'you have to pass server_hostname when using ssl'): + + self.loop.run_until_complete(coro) + + +class UnixReadPipeTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.pipe = mock.Mock(spec_set=io.RawIOBase) + self.pipe.fileno.return_value = 5 + + blocking_patcher = mock.patch('asyncio.unix_events._set_nonblocking') + blocking_patcher.start() + self.addCleanup(blocking_patcher.stop) + + fstat_patcher = mock.patch('os.fstat') + m_fstat = fstat_patcher.start() + st = mock.Mock() + st.st_mode = stat.S_IFIFO + m_fstat.return_value = st + self.addCleanup(fstat_patcher.stop) + + def read_pipe_transport(self, waiter=None): + transport = unix_events._UnixReadPipeTransport(self.loop, self.pipe, + self.protocol, + waiter=waiter) + self.addCleanup(close_pipe_transport, transport) + return transport + + def test_ctor(self): + waiter = asyncio.Future(loop=self.loop) + tr = self.read_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) + self.loop.assert_reader(5, tr._read_ready) + self.assertIsNone(waiter.result()) + + @mock.patch('os.read') + def test__read_ready(self, m_read): + tr = self.read_pipe_transport() + m_read.return_value = b'data' + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + self.protocol.data_received.assert_called_with(b'data') + + @mock.patch('os.read') + def test__read_ready_eof(self, m_read): + tr = self.read_pipe_transport() + m_read.return_value = b'' + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.eof_received.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + @mock.patch('os.read') + def test__read_ready_blocked(self, m_read): + tr = self.read_pipe_transport() + m_read.side_effect = BlockingIOError + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.data_received.called) + + @mock.patch('asyncio.log.logger.error') + @mock.patch('os.read') + def test__read_ready_error(self, m_read, m_logexc): + tr = self.read_pipe_transport() + err = OSError() + m_read.side_effect = err + tr._close = mock.Mock() + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + tr._close.assert_called_with(err) + m_logexc.assert_called_with( + test_utils.MockPattern( + 'Fatal read error on pipe transport' + '\nprotocol:.*\ntransport:.*'), + exc_info=(OSError, MOCK_ANY, MOCK_ANY)) + + @mock.patch('os.read') + def test_pause_reading(self, m_read): + tr = self.read_pipe_transport() + m = mock.Mock() + self.loop.add_reader(5, m) + tr.pause_reading() + self.assertFalse(self.loop.readers) + + @mock.patch('os.read') + def test_resume_reading(self, m_read): + tr = self.read_pipe_transport() + tr.resume_reading() + self.loop.assert_reader(5, tr._read_ready) + + @mock.patch('os.read') + def test_close(self, m_read): + tr = self.read_pipe_transport() + tr._close = mock.Mock() + tr.close() + tr._close.assert_called_with(None) + + @mock.patch('os.read') + def test_close_already_closing(self, m_read): + tr = self.read_pipe_transport() + tr._closing = True + tr._close = mock.Mock() + tr.close() + self.assertFalse(tr._close.called) + + @mock.patch('os.read') + def test__close(self, m_read): + tr = self.read_pipe_transport() + err = object() + tr._close(err) + self.assertTrue(tr.is_closing()) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(err) + + def test__call_connection_lost(self): + tr = self.read_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = None + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test__call_connection_lost_with_err(self): + tr = self.read_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = OSError() + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + +class UnixWritePipeTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.BaseProtocol) + self.pipe = mock.Mock(spec_set=io.RawIOBase) + self.pipe.fileno.return_value = 5 + + blocking_patcher = mock.patch('asyncio.unix_events._set_nonblocking') + blocking_patcher.start() + self.addCleanup(blocking_patcher.stop) + + fstat_patcher = mock.patch('os.fstat') + m_fstat = fstat_patcher.start() + st = mock.Mock() + st.st_mode = stat.S_IFSOCK + m_fstat.return_value = st + self.addCleanup(fstat_patcher.stop) + + def write_pipe_transport(self, waiter=None): + transport = unix_events._UnixWritePipeTransport(self.loop, self.pipe, + self.protocol, + waiter=waiter) + self.addCleanup(close_pipe_transport, transport) + return transport + + def test_ctor(self): + waiter = asyncio.Future(loop=self.loop) + tr = self.write_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) + self.loop.assert_reader(5, tr._read_ready) + self.assertEqual(None, waiter.result()) + + def test_can_write_eof(self): + tr = self.write_pipe_transport() + self.assertTrue(tr.can_write_eof()) + + @mock.patch('os.write') + def test_write(self, m_write): + tr = self.write_pipe_transport() + m_write.return_value = 4 + tr.write(b'data') + m_write.assert_called_with(5, b'data') + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + + @mock.patch('os.write') + def test_write_no_data(self, m_write): + tr = self.write_pipe_transport() + tr.write(b'') + self.assertFalse(m_write.called) + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(b''), tr._buffer) + + @mock.patch('os.write') + def test_write_partial(self, m_write): + tr = self.write_pipe_transport() + m_write.return_value = 2 + tr.write(b'data') + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'ta'), tr._buffer) + + @mock.patch('os.write') + def test_write_buffer(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'previous') + tr.write(b'data') + self.assertFalse(m_write.called) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'previousdata'), tr._buffer) + + @mock.patch('os.write') + def test_write_again(self, m_write): + tr = self.write_pipe_transport() + m_write.side_effect = BlockingIOError() + tr.write(b'data') + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('asyncio.unix_events.logger') + @mock.patch('os.write') + def test_write_err(self, m_write, m_log): + tr = self.write_pipe_transport() + err = OSError() + m_write.side_effect = err + tr._fatal_error = mock.Mock() + tr.write(b'data') + m_write.assert_called_with(5, b'data') + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + tr._fatal_error.assert_called_with( + err, + 'Fatal write error on pipe transport') + self.assertEqual(1, tr._conn_lost) + + tr.write(b'data') + self.assertEqual(2, tr._conn_lost) + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + # This is a bit overspecified. :-( + m_log.warning.assert_called_with( + 'pipe closed by peer or os.write(pipe, data) raised exception.') + tr.close() + + @mock.patch('os.write') + def test_write_close(self, m_write): + tr = self.write_pipe_transport() + tr._read_ready() # pipe was closed by peer + + tr.write(b'data') + self.assertEqual(tr._conn_lost, 1) + tr.write(b'data') + self.assertEqual(tr._conn_lost, 2) + + def test__read_ready(self): + tr = self.write_pipe_transport() + tr._read_ready() + self.assertFalse(self.loop.readers) + self.assertFalse(self.loop.writers) + self.assertTrue(tr.is_closing()) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + @mock.patch('os.write') + def test__write_ready(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 4 + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_partial(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 3 + tr._write_ready() + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'a'), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_again(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.side_effect = BlockingIOError() + tr._write_ready() + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_empty(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 0 + tr._write_ready() + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('asyncio.log.logger.error') + @mock.patch('os.write') + def test__write_ready_err(self, m_write, m_logexc): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.side_effect = err = OSError() + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertFalse(self.loop.readers) + self.assertEqual(bytearray(), tr._buffer) + self.assertTrue(tr.is_closing()) + m_logexc.assert_called_with( + test_utils.MockPattern( + 'Fatal write error on pipe transport' + '\nprotocol:.*\ntransport:.*'), + exc_info=(OSError, MOCK_ANY, MOCK_ANY)) + self.assertEqual(1, tr._conn_lost) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(err) + + @mock.patch('os.write') + def test__write_ready_closing(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._closing = True + tr._buffer = bytearray(b'data') + m_write.return_value = 4 + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertFalse(self.loop.readers) + self.assertEqual(bytearray(), tr._buffer) + self.protocol.connection_lost.assert_called_with(None) + self.pipe.close.assert_called_with() + + @mock.patch('os.write') + def test_abort(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + self.loop.add_reader(5, tr._read_ready) + tr._buffer = [b'da', b'ta'] + tr.abort() + self.assertFalse(m_write.called) + self.assertFalse(self.loop.readers) + self.assertFalse(self.loop.writers) + self.assertEqual([], tr._buffer) + self.assertTrue(tr.is_closing()) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test__call_connection_lost(self): + tr = self.write_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = None + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test__call_connection_lost_with_err(self): + tr = self.write_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = OSError() + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test_close(self): + tr = self.write_pipe_transport() + tr.write_eof = mock.Mock() + tr.close() + tr.write_eof.assert_called_with() + + # closing the transport twice must not fail + tr.close() + + def test_close_closing(self): + tr = self.write_pipe_transport() + tr.write_eof = mock.Mock() + tr._closing = True + tr.close() + self.assertFalse(tr.write_eof.called) + + def test_write_eof(self): + tr = self.write_pipe_transport() + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test_write_eof_pending(self): + tr = self.write_pipe_transport() + tr._buffer = [b'data'] + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.protocol.connection_lost.called) + + +class AbstractChildWatcherTests(unittest.TestCase): + + def test_not_implemented(self): + f = mock.Mock() + watcher = asyncio.AbstractChildWatcher() + self.assertRaises( + NotImplementedError, watcher.add_child_handler, f, f) + self.assertRaises( + NotImplementedError, watcher.remove_child_handler, f) + self.assertRaises( + NotImplementedError, watcher.attach_loop, f) + self.assertRaises( + NotImplementedError, watcher.close) + self.assertRaises( + NotImplementedError, watcher.__enter__) + self.assertRaises( + NotImplementedError, watcher.__exit__, f, f, f) + + +class BaseChildWatcherTests(unittest.TestCase): + + def test_not_implemented(self): + f = mock.Mock() + watcher = unix_events.BaseChildWatcher() + self.assertRaises( + NotImplementedError, watcher._do_waitpid, f) + + +WaitPidMocks = collections.namedtuple("WaitPidMocks", + ("waitpid", + "WIFEXITED", + "WIFSIGNALED", + "WEXITSTATUS", + "WTERMSIG", + )) + + +class ChildWatcherTestsMixin: + + ignore_warnings = mock.patch.object(log.logger, "warning") + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.running = False + self.zombies = {} + + with mock.patch.object( + self.loop, "add_signal_handler") as self.m_add_signal_handler: + self.watcher = self.create_watcher() + self.watcher.attach_loop(self.loop) + + def waitpid(self, pid, flags): + if isinstance(self.watcher, asyncio.SafeChildWatcher) or pid != -1: + self.assertGreater(pid, 0) + try: + if pid < 0: + return self.zombies.popitem() + else: + return pid, self.zombies.pop(pid) + except KeyError: + pass + if self.running: + return 0, 0 + else: + raise ChildProcessError() + + def add_zombie(self, pid, returncode): + self.zombies[pid] = returncode + 32768 + + def WIFEXITED(self, status): + return status >= 32768 + + def WIFSIGNALED(self, status): + return 32700 < status < 32768 + + def WEXITSTATUS(self, status): + self.assertTrue(self.WIFEXITED(status)) + return status - 32768 + + def WTERMSIG(self, status): + self.assertTrue(self.WIFSIGNALED(status)) + return 32768 - status + + def test_create_watcher(self): + self.m_add_signal_handler.assert_called_once_with( + signal.SIGCHLD, self.watcher._sig_chld) + + def waitpid_mocks(func): + def wrapped_func(self): + def patch(target, wrapper): + return mock.patch(target, wraps=wrapper, + new_callable=mock.Mock) + + with patch('os.WTERMSIG', self.WTERMSIG) as m_WTERMSIG, \ + patch('os.WEXITSTATUS', self.WEXITSTATUS) as m_WEXITSTATUS, \ + patch('os.WIFSIGNALED', self.WIFSIGNALED) as m_WIFSIGNALED, \ + patch('os.WIFEXITED', self.WIFEXITED) as m_WIFEXITED, \ + patch('os.waitpid', self.waitpid) as m_waitpid: + func(self, WaitPidMocks(m_waitpid, + m_WIFEXITED, m_WIFSIGNALED, + m_WEXITSTATUS, m_WTERMSIG, + )) + return wrapped_func + + @waitpid_mocks + def test_sigchld(self, m): + # register a child + callback = mock.Mock() + + with self.watcher: + self.running = True + self.watcher.add_child_handler(42, callback, 9, 10, 14) + + self.assertFalse(callback.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # child is running + self.watcher._sig_chld() + + self.assertFalse(callback.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # child terminates (returncode 12) + self.running = False + self.add_zombie(42, 12) + self.watcher._sig_chld() + + self.assertTrue(m.WIFEXITED.called) + self.assertTrue(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + callback.assert_called_once_with(42, 12, 9, 10, 14) + + m.WIFSIGNALED.reset_mock() + m.WIFEXITED.reset_mock() + m.WEXITSTATUS.reset_mock() + callback.reset_mock() + + # ensure that the child is effectively reaped + self.add_zombie(42, 13) + with self.ignore_warnings: + self.watcher._sig_chld() + + self.assertFalse(callback.called) + self.assertFalse(m.WTERMSIG.called) + + m.WIFSIGNALED.reset_mock() + m.WIFEXITED.reset_mock() + m.WEXITSTATUS.reset_mock() + + # sigchld called again + self.zombies.clear() + self.watcher._sig_chld() + + self.assertFalse(callback.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + @waitpid_mocks + def test_sigchld_two_children(self, m): + callback1 = mock.Mock() + callback2 = mock.Mock() + + # register child 1 + with self.watcher: + self.running = True + self.watcher.add_child_handler(43, callback1, 7, 8) + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # register child 2 + with self.watcher: + self.watcher.add_child_handler(44, callback2, 147, 18) + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # children are running + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # child 1 terminates (signal 3) + self.add_zombie(43, -3) + self.watcher._sig_chld() + + callback1.assert_called_once_with(43, -3, 7, 8) + self.assertFalse(callback2.called) + self.assertTrue(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertTrue(m.WTERMSIG.called) + + m.WIFSIGNALED.reset_mock() + m.WIFEXITED.reset_mock() + m.WTERMSIG.reset_mock() + callback1.reset_mock() + + # child 2 still running + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # child 2 terminates (code 108) + self.add_zombie(44, 108) + self.running = False + self.watcher._sig_chld() + + callback2.assert_called_once_with(44, 108, 147, 18) + self.assertFalse(callback1.called) + self.assertTrue(m.WIFEXITED.called) + self.assertTrue(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + m.WIFSIGNALED.reset_mock() + m.WIFEXITED.reset_mock() + m.WEXITSTATUS.reset_mock() + callback2.reset_mock() + + # ensure that the children are effectively reaped + self.add_zombie(43, 14) + self.add_zombie(44, 15) + with self.ignore_warnings: + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WTERMSIG.called) + + m.WIFSIGNALED.reset_mock() + m.WIFEXITED.reset_mock() + m.WEXITSTATUS.reset_mock() + + # sigchld called again + self.zombies.clear() + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + @waitpid_mocks + def test_sigchld_two_children_terminating_together(self, m): + callback1 = mock.Mock() + callback2 = mock.Mock() + + # register child 1 + with self.watcher: + self.running = True + self.watcher.add_child_handler(45, callback1, 17, 8) + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # register child 2 + with self.watcher: + self.watcher.add_child_handler(46, callback2, 1147, 18) + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # children are running + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # child 1 terminates (code 78) + # child 2 terminates (signal 5) + self.add_zombie(45, 78) + self.add_zombie(46, -5) + self.running = False + self.watcher._sig_chld() + + callback1.assert_called_once_with(45, 78, 17, 8) + callback2.assert_called_once_with(46, -5, 1147, 18) + self.assertTrue(m.WIFSIGNALED.called) + self.assertTrue(m.WIFEXITED.called) + self.assertTrue(m.WEXITSTATUS.called) + self.assertTrue(m.WTERMSIG.called) + + m.WIFSIGNALED.reset_mock() + m.WIFEXITED.reset_mock() + m.WTERMSIG.reset_mock() + m.WEXITSTATUS.reset_mock() + callback1.reset_mock() + callback2.reset_mock() + + # ensure that the children are effectively reaped + self.add_zombie(45, 14) + self.add_zombie(46, 15) + with self.ignore_warnings: + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WTERMSIG.called) + + @waitpid_mocks + def test_sigchld_race_condition(self, m): + # register a child + callback = mock.Mock() + + with self.watcher: + # child terminates before being registered + self.add_zombie(50, 4) + self.watcher._sig_chld() + + self.watcher.add_child_handler(50, callback, 1, 12) + + callback.assert_called_once_with(50, 4, 1, 12) + callback.reset_mock() + + # ensure that the child is effectively reaped + self.add_zombie(50, -1) + with self.ignore_warnings: + self.watcher._sig_chld() + + self.assertFalse(callback.called) + + @waitpid_mocks + def test_sigchld_replace_handler(self, m): + callback1 = mock.Mock() + callback2 = mock.Mock() + + # register a child + with self.watcher: + self.running = True + self.watcher.add_child_handler(51, callback1, 19) + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # register the same child again + with self.watcher: + self.watcher.add_child_handler(51, callback2, 21) + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # child terminates (signal 8) + self.running = False + self.add_zombie(51, -8) + self.watcher._sig_chld() + + callback2.assert_called_once_with(51, -8, 21) + self.assertFalse(callback1.called) + self.assertTrue(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertTrue(m.WTERMSIG.called) + + m.WIFSIGNALED.reset_mock() + m.WIFEXITED.reset_mock() + m.WTERMSIG.reset_mock() + callback2.reset_mock() + + # ensure that the child is effectively reaped + self.add_zombie(51, 13) + with self.ignore_warnings: + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(m.WTERMSIG.called) + + @waitpid_mocks + def test_sigchld_remove_handler(self, m): + callback = mock.Mock() + + # register a child + with self.watcher: + self.running = True + self.watcher.add_child_handler(52, callback, 1984) + + self.assertFalse(callback.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # unregister the child + self.watcher.remove_child_handler(52) + + self.assertFalse(callback.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # child terminates (code 99) + self.running = False + self.add_zombie(52, 99) + with self.ignore_warnings: + self.watcher._sig_chld() + + self.assertFalse(callback.called) + + @waitpid_mocks + def test_sigchld_unknown_status(self, m): + callback = mock.Mock() + + # register a child + with self.watcher: + self.running = True + self.watcher.add_child_handler(53, callback, -19) + + self.assertFalse(callback.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # terminate with unknown status + self.zombies[53] = 1178 + self.running = False + self.watcher._sig_chld() + + callback.assert_called_once_with(53, 1178, -19) + self.assertTrue(m.WIFEXITED.called) + self.assertTrue(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + callback.reset_mock() + m.WIFEXITED.reset_mock() + m.WIFSIGNALED.reset_mock() + + # ensure that the child is effectively reaped + self.add_zombie(53, 101) + with self.ignore_warnings: + self.watcher._sig_chld() + + self.assertFalse(callback.called) + + @waitpid_mocks + def test_remove_child_handler(self, m): + callback1 = mock.Mock() + callback2 = mock.Mock() + callback3 = mock.Mock() + + # register children + with self.watcher: + self.running = True + self.watcher.add_child_handler(54, callback1, 1) + self.watcher.add_child_handler(55, callback2, 2) + self.watcher.add_child_handler(56, callback3, 3) + + # remove child handler 1 + self.assertTrue(self.watcher.remove_child_handler(54)) + + # remove child handler 2 multiple times + self.assertTrue(self.watcher.remove_child_handler(55)) + self.assertFalse(self.watcher.remove_child_handler(55)) + self.assertFalse(self.watcher.remove_child_handler(55)) + + # all children terminate + self.add_zombie(54, 0) + self.add_zombie(55, 1) + self.add_zombie(56, 2) + self.running = False + with self.ignore_warnings: + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + callback3.assert_called_once_with(56, 2, 3) + + @waitpid_mocks + def test_sigchld_unhandled_exception(self, m): + callback = mock.Mock() + + # register a child + with self.watcher: + self.running = True + self.watcher.add_child_handler(57, callback) + + # raise an exception + m.waitpid.side_effect = ValueError + + with mock.patch.object(log.logger, + 'error') as m_error: + + self.assertEqual(self.watcher._sig_chld(), None) + self.assertTrue(m_error.called) + + @waitpid_mocks + def test_sigchld_child_reaped_elsewhere(self, m): + # register a child + callback = mock.Mock() + + with self.watcher: + self.running = True + self.watcher.add_child_handler(58, callback) + + self.assertFalse(callback.called) + self.assertFalse(m.WIFEXITED.called) + self.assertFalse(m.WIFSIGNALED.called) + self.assertFalse(m.WEXITSTATUS.called) + self.assertFalse(m.WTERMSIG.called) + + # child terminates + self.running = False + self.add_zombie(58, 4) + + # waitpid is called elsewhere + os.waitpid(58, os.WNOHANG) + + m.waitpid.reset_mock() + + # sigchld + with self.ignore_warnings: + self.watcher._sig_chld() + + if isinstance(self.watcher, asyncio.FastChildWatcher): + # here the FastChildWatche enters a deadlock + # (there is no way to prevent it) + self.assertFalse(callback.called) + else: + callback.assert_called_once_with(58, 255) + + @waitpid_mocks + def test_sigchld_unknown_pid_during_registration(self, m): + # register two children + callback1 = mock.Mock() + callback2 = mock.Mock() + + with self.ignore_warnings, self.watcher: + self.running = True + # child 1 terminates + self.add_zombie(591, 7) + # an unknown child terminates + self.add_zombie(593, 17) + + self.watcher._sig_chld() + + self.watcher.add_child_handler(591, callback1) + self.watcher.add_child_handler(592, callback2) + + callback1.assert_called_once_with(591, 7) + self.assertFalse(callback2.called) + + @waitpid_mocks + def test_set_loop(self, m): + # register a child + callback = mock.Mock() + + with self.watcher: + self.running = True + self.watcher.add_child_handler(60, callback) + + # attach a new loop + old_loop = self.loop + self.loop = self.new_test_loop() + patch = mock.patch.object + + with patch(old_loop, "remove_signal_handler") as m_old_remove, \ + patch(self.loop, "add_signal_handler") as m_new_add: + + self.watcher.attach_loop(self.loop) + + m_old_remove.assert_called_once_with( + signal.SIGCHLD) + m_new_add.assert_called_once_with( + signal.SIGCHLD, self.watcher._sig_chld) + + # child terminates + self.running = False + self.add_zombie(60, 9) + self.watcher._sig_chld() + + callback.assert_called_once_with(60, 9) + + @waitpid_mocks + def test_set_loop_race_condition(self, m): + # register 3 children + callback1 = mock.Mock() + callback2 = mock.Mock() + callback3 = mock.Mock() + + with self.watcher: + self.running = True + self.watcher.add_child_handler(61, callback1) + self.watcher.add_child_handler(62, callback2) + self.watcher.add_child_handler(622, callback3) + + # detach the loop + old_loop = self.loop + self.loop = None + + with mock.patch.object( + old_loop, "remove_signal_handler") as m_remove_signal_handler: + + with self.assertWarnsRegex( + RuntimeWarning, 'A loop is being detached'): + self.watcher.attach_loop(None) + + m_remove_signal_handler.assert_called_once_with( + signal.SIGCHLD) + + # child 1 & 2 terminate + self.add_zombie(61, 11) + self.add_zombie(62, -5) + + # SIGCHLD was not caught + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + self.assertFalse(callback3.called) + + # attach a new loop + self.loop = self.new_test_loop() + + with mock.patch.object( + self.loop, "add_signal_handler") as m_add_signal_handler: + + self.watcher.attach_loop(self.loop) + + m_add_signal_handler.assert_called_once_with( + signal.SIGCHLD, self.watcher._sig_chld) + callback1.assert_called_once_with(61, 11) # race condition! + callback2.assert_called_once_with(62, -5) # race condition! + self.assertFalse(callback3.called) + + callback1.reset_mock() + callback2.reset_mock() + + # child 3 terminates + self.running = False + self.add_zombie(622, 19) + self.watcher._sig_chld() + + self.assertFalse(callback1.called) + self.assertFalse(callback2.called) + callback3.assert_called_once_with(622, 19) + + @waitpid_mocks + def test_close(self, m): + # register two children + callback1 = mock.Mock() + + with self.watcher: + self.running = True + # child 1 terminates + self.add_zombie(63, 9) + # other child terminates + self.add_zombie(65, 18) + self.watcher._sig_chld() + + self.watcher.add_child_handler(63, callback1) + self.watcher.add_child_handler(64, callback1) + + self.assertEqual(len(self.watcher._callbacks), 1) + if isinstance(self.watcher, asyncio.FastChildWatcher): + self.assertEqual(len(self.watcher._zombies), 1) + + with mock.patch.object( + self.loop, + "remove_signal_handler") as m_remove_signal_handler: + + self.watcher.close() + + m_remove_signal_handler.assert_called_once_with( + signal.SIGCHLD) + self.assertFalse(self.watcher._callbacks) + if isinstance(self.watcher, asyncio.FastChildWatcher): + self.assertFalse(self.watcher._zombies) + + @waitpid_mocks + def test_add_child_handler_with_no_loop_attached(self, m): + callback = mock.Mock() + with self.create_watcher() as watcher: + with self.assertRaisesRegex( + RuntimeError, + 'the child watcher does not have a loop attached'): + watcher.add_child_handler(100, callback) + + +class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): + def create_watcher(self): + return asyncio.SafeChildWatcher() + + +class FastChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): + def create_watcher(self): + return asyncio.FastChildWatcher() + + +class PolicyTests(unittest.TestCase): + + def create_policy(self): + return asyncio.DefaultEventLoopPolicy() + + def test_get_child_watcher(self): + policy = self.create_policy() + self.assertIsNone(policy._watcher) + + watcher = policy.get_child_watcher() + self.assertIsInstance(watcher, asyncio.SafeChildWatcher) + + self.assertIs(policy._watcher, watcher) + + self.assertIs(watcher, policy.get_child_watcher()) + self.assertIsNone(watcher._loop) + + def test_get_child_watcher_after_set(self): + policy = self.create_policy() + watcher = asyncio.FastChildWatcher() + + policy.set_child_watcher(watcher) + self.assertIs(policy._watcher, watcher) + self.assertIs(watcher, policy.get_child_watcher()) + + def test_get_child_watcher_with_mainloop_existing(self): + policy = self.create_policy() + loop = policy.get_event_loop() + + self.assertIsNone(policy._watcher) + watcher = policy.get_child_watcher() + + self.assertIsInstance(watcher, asyncio.SafeChildWatcher) + self.assertIs(watcher._loop, loop) + + loop.close() + + def test_get_child_watcher_thread(self): + + def f(): + policy.set_event_loop(policy.new_event_loop()) + + self.assertIsInstance(policy.get_event_loop(), + asyncio.AbstractEventLoop) + watcher = policy.get_child_watcher() + + self.assertIsInstance(watcher, asyncio.SafeChildWatcher) + self.assertIsNone(watcher._loop) + + policy.get_event_loop().close() + + policy = self.create_policy() + + th = threading.Thread(target=f) + th.start() + th.join() + + def test_child_watcher_replace_mainloop_existing(self): + policy = self.create_policy() + loop = policy.get_event_loop() + + watcher = policy.get_child_watcher() + + self.assertIs(watcher._loop, loop) + + new_loop = policy.new_event_loop() + policy.set_event_loop(new_loop) + + self.assertIs(watcher._loop, new_loop) + + policy.set_event_loop(None) + + self.assertIs(watcher._loop, None) + + loop.close() + new_loop.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_windows_events.py b/thirdparty/asyncio/tests/test_windows_events.py new file mode 100644 index 0000000..1afcae1 --- /dev/null +++ b/thirdparty/asyncio/tests/test_windows_events.py @@ -0,0 +1,162 @@ +import os +import sys +import unittest +from unittest import mock + +if sys.platform != 'win32': + raise unittest.SkipTest('Windows only') + +import _winapi + +import asyncio +from asyncio import _overlapped +from asyncio import test_utils +from asyncio import windows_events + + +class UpperProto(asyncio.Protocol): + def __init__(self): + self.buf = [] + + def connection_made(self, trans): + self.trans = trans + + def data_received(self, data): + self.buf.append(data) + if b'\n' in data: + self.trans.write(b''.join(self.buf).upper()) + self.trans.close() + + +class ProactorTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.ProactorEventLoop() + self.set_event_loop(self.loop) + + def test_close(self): + a, b = self.loop._socketpair() + trans = self.loop._make_socket_transport(a, asyncio.Protocol()) + f = asyncio.ensure_future(self.loop.sock_recv(b, 100)) + trans.close() + self.loop.run_until_complete(f) + self.assertEqual(f.result(), b'') + b.close() + + def test_double_bind(self): + ADDRESS = r'\\.\pipe\test_double_bind-%s' % os.getpid() + server1 = windows_events.PipeServer(ADDRESS) + with self.assertRaises(PermissionError): + windows_events.PipeServer(ADDRESS) + server1.close() + + def test_pipe(self): + res = self.loop.run_until_complete(self._test_pipe()) + self.assertEqual(res, 'done') + + def _test_pipe(self): + ADDRESS = r'\\.\pipe\_test_pipe-%s' % os.getpid() + + with self.assertRaises(FileNotFoundError): + yield from self.loop.create_pipe_connection( + asyncio.Protocol, ADDRESS) + + [server] = yield from self.loop.start_serving_pipe( + UpperProto, ADDRESS) + self.assertIsInstance(server, windows_events.PipeServer) + + clients = [] + for i in range(5): + stream_reader = asyncio.StreamReader(loop=self.loop) + protocol = asyncio.StreamReaderProtocol(stream_reader, + loop=self.loop) + trans, proto = yield from self.loop.create_pipe_connection( + lambda: protocol, ADDRESS) + self.assertIsInstance(trans, asyncio.Transport) + self.assertEqual(protocol, proto) + clients.append((stream_reader, trans)) + + for i, (r, w) in enumerate(clients): + w.write('lower-{}\n'.format(i).encode()) + + for i, (r, w) in enumerate(clients): + response = yield from r.readline() + self.assertEqual(response, 'LOWER-{}\n'.format(i).encode()) + w.close() + + server.close() + + with self.assertRaises(FileNotFoundError): + yield from self.loop.create_pipe_connection( + asyncio.Protocol, ADDRESS) + + return 'done' + + def test_connect_pipe_cancel(self): + exc = OSError() + exc.winerror = _overlapped.ERROR_PIPE_BUSY + with mock.patch.object(_overlapped, 'ConnectPipe', side_effect=exc) as connect: + coro = self.loop._proactor.connect_pipe('pipe_address') + task = self.loop.create_task(coro) + + # check that it's possible to cancel connect_pipe() + task.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(task) + + def test_wait_for_handle(self): + event = _overlapped.CreateEvent(None, True, False, None) + self.addCleanup(_winapi.CloseHandle, event) + + # Wait for unset event with 0.5s timeout; + # result should be False at timeout + fut = self.loop._proactor.wait_for_handle(event, 0.5) + start = self.loop.time() + done = self.loop.run_until_complete(fut) + elapsed = self.loop.time() - start + + self.assertEqual(done, False) + self.assertFalse(fut.result()) + self.assertTrue(0.48 < elapsed < 0.9, elapsed) + + _overlapped.SetEvent(event) + + # Wait for set event; + # result should be True immediately + fut = self.loop._proactor.wait_for_handle(event, 10) + start = self.loop.time() + done = self.loop.run_until_complete(fut) + elapsed = self.loop.time() - start + + self.assertEqual(done, True) + self.assertTrue(fut.result()) + self.assertTrue(0 <= elapsed < 0.3, elapsed) + + # asyncio issue #195: cancelling a done _WaitHandleFuture + # must not crash + fut.cancel() + + def test_wait_for_handle_cancel(self): + event = _overlapped.CreateEvent(None, True, False, None) + self.addCleanup(_winapi.CloseHandle, event) + + # Wait for unset event with a cancelled future; + # CancelledError should be raised immediately + fut = self.loop._proactor.wait_for_handle(event, 10) + fut.cancel() + start = self.loop.time() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(fut) + elapsed = self.loop.time() - start + self.assertTrue(0 <= elapsed < 0.1, elapsed) + + # asyncio issue #195: cancelling a _WaitHandleFuture twice + # must not crash + fut = self.loop._proactor.wait_for_handle(event) + fut.cancel() + fut.cancel() + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tests/test_windows_utils.py b/thirdparty/asyncio/tests/test_windows_utils.py new file mode 100644 index 0000000..d48b8bc --- /dev/null +++ b/thirdparty/asyncio/tests/test_windows_utils.py @@ -0,0 +1,182 @@ +"""Tests for window_utils""" + +import socket +import sys +import unittest +import warnings +from unittest import mock + +if sys.platform != 'win32': + raise unittest.SkipTest('Windows only') + +import _winapi + +from asyncio import _overlapped +from asyncio import windows_utils +try: + from test import support +except ImportError: + from asyncio import test_support as support + + +class WinsocketpairTests(unittest.TestCase): + + def check_winsocketpair(self, ssock, csock): + csock.send(b'xxx') + self.assertEqual(b'xxx', ssock.recv(1024)) + csock.close() + ssock.close() + + def test_winsocketpair(self): + ssock, csock = windows_utils.socketpair() + self.check_winsocketpair(ssock, csock) + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_winsocketpair_ipv6(self): + ssock, csock = windows_utils.socketpair(family=socket.AF_INET6) + self.check_winsocketpair(ssock, csock) + + @unittest.skipIf(hasattr(socket, 'socketpair'), + 'socket.socketpair is available') + @mock.patch('asyncio.windows_utils.socket') + def test_winsocketpair_exc(self, m_socket): + m_socket.AF_INET = socket.AF_INET + m_socket.SOCK_STREAM = socket.SOCK_STREAM + m_socket.socket.return_value.getsockname.return_value = ('', 12345) + m_socket.socket.return_value.accept.return_value = object(), object() + m_socket.socket.return_value.connect.side_effect = OSError() + + self.assertRaises(OSError, windows_utils.socketpair) + + def test_winsocketpair_invalid_args(self): + self.assertRaises(ValueError, + windows_utils.socketpair, family=socket.AF_UNSPEC) + self.assertRaises(ValueError, + windows_utils.socketpair, type=socket.SOCK_DGRAM) + self.assertRaises(ValueError, + windows_utils.socketpair, proto=1) + + @unittest.skipIf(hasattr(socket, 'socketpair'), + 'socket.socketpair is available') + @mock.patch('asyncio.windows_utils.socket') + def test_winsocketpair_close(self, m_socket): + m_socket.AF_INET = socket.AF_INET + m_socket.SOCK_STREAM = socket.SOCK_STREAM + sock = mock.Mock() + m_socket.socket.return_value = sock + sock.bind.side_effect = OSError + self.assertRaises(OSError, windows_utils.socketpair) + self.assertTrue(sock.close.called) + + +class PipeTests(unittest.TestCase): + + def test_pipe_overlapped(self): + h1, h2 = windows_utils.pipe(overlapped=(True, True)) + try: + ov1 = _overlapped.Overlapped() + self.assertFalse(ov1.pending) + self.assertEqual(ov1.error, 0) + + ov1.ReadFile(h1, 100) + self.assertTrue(ov1.pending) + self.assertEqual(ov1.error, _winapi.ERROR_IO_PENDING) + ERROR_IO_INCOMPLETE = 996 + try: + ov1.getresult() + except OSError as e: + self.assertEqual(e.winerror, ERROR_IO_INCOMPLETE) + else: + raise RuntimeError('expected ERROR_IO_INCOMPLETE') + + ov2 = _overlapped.Overlapped() + self.assertFalse(ov2.pending) + self.assertEqual(ov2.error, 0) + + ov2.WriteFile(h2, b"hello") + self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING}) + + res = _winapi.WaitForMultipleObjects([ov2.event], False, 100) + self.assertEqual(res, _winapi.WAIT_OBJECT_0) + + self.assertFalse(ov1.pending) + self.assertEqual(ov1.error, ERROR_IO_INCOMPLETE) + self.assertFalse(ov2.pending) + self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING}) + self.assertEqual(ov1.getresult(), b"hello") + finally: + _winapi.CloseHandle(h1) + _winapi.CloseHandle(h2) + + def test_pipe_handle(self): + h, _ = windows_utils.pipe(overlapped=(True, True)) + _winapi.CloseHandle(_) + p = windows_utils.PipeHandle(h) + self.assertEqual(p.fileno(), h) + self.assertEqual(p.handle, h) + + # check garbage collection of p closes handle + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", ResourceWarning) + del p + support.gc_collect() + try: + _winapi.CloseHandle(h) + except OSError as e: + self.assertEqual(e.winerror, 6) # ERROR_INVALID_HANDLE + else: + raise RuntimeError('expected ERROR_INVALID_HANDLE') + + +class PopenTests(unittest.TestCase): + + def test_popen(self): + command = r"""if 1: + import sys + s = sys.stdin.readline() + sys.stdout.write(s.upper()) + sys.stderr.write('stderr') + """ + msg = b"blah\n" + + p = windows_utils.Popen([sys.executable, '-c', command], + stdin=windows_utils.PIPE, + stdout=windows_utils.PIPE, + stderr=windows_utils.PIPE) + + for f in [p.stdin, p.stdout, p.stderr]: + self.assertIsInstance(f, windows_utils.PipeHandle) + + ovin = _overlapped.Overlapped() + ovout = _overlapped.Overlapped() + overr = _overlapped.Overlapped() + + ovin.WriteFile(p.stdin.handle, msg) + ovout.ReadFile(p.stdout.handle, 100) + overr.ReadFile(p.stderr.handle, 100) + + events = [ovin.event, ovout.event, overr.event] + # Super-long timeout for slow buildbots. + res = _winapi.WaitForMultipleObjects(events, True, 10000) + self.assertEqual(res, _winapi.WAIT_OBJECT_0) + self.assertFalse(ovout.pending) + self.assertFalse(overr.pending) + self.assertFalse(ovin.pending) + + self.assertEqual(ovin.getresult(), len(msg)) + out = ovout.getresult().rstrip() + err = overr.getresult().rstrip() + + self.assertGreater(len(out), 0) + self.assertGreater(len(err), 0) + # allow for partial reads... + self.assertTrue(msg.upper().rstrip().startswith(out)) + self.assertTrue(b"stderr".startswith(err)) + + # The context manager calls wait() and closes resources + with p: + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/thirdparty/asyncio/tox.ini b/thirdparty/asyncio/tox.ini new file mode 100644 index 0000000..3030441 --- /dev/null +++ b/thirdparty/asyncio/tox.ini @@ -0,0 +1,21 @@ +[tox] +envlist = py33,py34,py3_release + +[testenv] +deps= + aiotest +# Run tests in debug mode +setenv = + PYTHONASYNCIODEBUG = 1 +commands= + python -Wd runtests.py -r {posargs} + python -Wd run_aiotest.py -r {posargs} + +[testenv:py3_release] +# Run tests in release mode +setenv = + PYTHONASYNCIODEBUG = +basepython = python3 + +[testenv:py35] +basepython = python3.5 diff --git a/thirdparty/asyncio/update_asyncio.sh b/thirdparty/asyncio/update_asyncio.sh new file mode 100644 index 0000000..7ef3d72 --- /dev/null +++ b/thirdparty/asyncio/update_asyncio.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +PYTHON=${1-$HOME/cpython} + +if [ ! -d $PYTHON ] +then + echo Bad destination $PYTHON + exit 1 +fi + +if [ ! -f asyncio/__init__.py ] +then + echo Bad current directory + exit 1 +fi + +echo "Sync from $PYTHON to $ASYNCIO" +set -e -x +echo + +cp $PYTHON/Lib/asyncio/*.py asyncio/ +cp $PYTHON/Lib/test/test_asyncio/test_*.py tests/ +echo + +git status diff --git a/thirdparty/asyncio/update_stdlib.sh b/thirdparty/asyncio/update_stdlib.sh new file mode 100644 index 0000000..14d5f9a --- /dev/null +++ b/thirdparty/asyncio/update_stdlib.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Script to copy asyncio files to the standard library tree. +# Optional argument is the root of the Python 3.4 tree. +# Assumes you have already created Lib/asyncio and +# Lib/test/test_asyncio in the destination tree. + +CPYTHON=${1-$HOME/cpython} + +if [ ! -d $CPYTHON ] +then + echo Bad destination $CPYTHON + exit 1 +fi + +if [ ! -f asyncio/__init__.py ] +then + echo Bad current directory + exit 1 +fi + +maybe_copy() +{ + SRC=$1 + DST=$CPYTHON/$2 + if cmp $DST $SRC + then + return + fi + echo ======== $SRC === $DST ======== + diff -u $DST $SRC + echo -n "Copy $SRC? [y/N/back] " + read X + case $X in + [yY]*) echo Copying $SRC; cp $SRC $DST;; + back) echo Copying TO $SRC; cp $DST $SRC;; + *) echo Not copying $SRC;; + esac +} + +for i in `(cd asyncio && ls *.py)` +do + if [ $i == test_support.py ] + then + continue + fi + + if [ $i == selectors.py ] + then + if [ "`(cd $CPYTHON; hg branch)`" == "3.4" ] + then + echo "Destination is 3.4 branch -- ignoring selectors.py" + else + maybe_copy asyncio/$i Lib/$i + fi + else + maybe_copy asyncio/$i Lib/asyncio/$i + fi +done + +for i in `(cd tests && ls *.py *.pem)` +do + if [ $i == test_selectors.py ] + then + continue + fi + if [ $i == test_pep492.py ] + then + if [ "`(cd $CPYTHON; hg branch)`" == "3.4" ] + then + echo "Destination is 3.4 branch -- ignoring test_pep492.py" + continue + fi + fi + maybe_copy tests/$i Lib/test/test_asyncio/$i +done + +maybe_copy overlapped.c Modules/overlapped.c