change from async to compat

conftest.py

@@ -4,7 +4,10 @@ from web3.providers.tester import (
     EthereumTesterProvider,
     TestRPCProvider,
 )
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+    sleep,
+)
 from web3.main import Web3
 
 
@@ -48,9 +51,9 @@ def skip_if_testrpc():
 def wait_for_miner_start():
     def _wait_for_miner_start(web3, timeout=60):
         poll_delay_counter = PollDelayCounter()
-        with async.Timeout(timeout) as timeout:
+        with Timeout(timeout) as timeout:
             while not web3.eth.mining or not web3.eth.hashrate:
-                async.sleep(poll_delay_counter())
+                sleep(poll_delay_counter())
                 timeout.check()
     return _wait_for_miner_start
 
@@ -59,13 +62,13 @@ def wait_for_miner_start():
 def wait_for_block():
     def _wait_for_block(web3, block_number=1, timeout=60 * 10):
         poll_delay_counter = PollDelayCounter()
-        with async.Timeout(timeout) as timeout:
+        with Timeout(timeout) as timeout:
             while True:
                 if web3.eth.blockNumber >= block_number:
                     break
                 if isinstance(web3.currentProvider, (TestRPCProvider, EthereumTesterProvider)):
                     web3._requestManager.request_blocking("evm_mine", [])
-                async.async.sleep(poll_delay_counter())
+                sleep(poll_delay_counter())
                 timeout.check()
     return _wait_for_block
 
@@ -74,12 +77,12 @@ def wait_for_block():
 def wait_for_transaction():
     def _wait_for_transaction(web3, txn_hash, timeout=120):
         poll_delay_counter = PollDelayCounter()
-        with async.Timeout(timeout) as timeout:
+        with Timeout(timeout) as timeout:
             while True:
                 txn_receipt = web3.eth.getTransactionReceipt(txn_hash)
                 if txn_receipt is not None:
                     break
-                async.sleep(poll_delay_counter())
+                sleep(poll_delay_counter())
                 timeout.check()
 
         return txn_receipt

docs/quickstart.rst

@@ -20,6 +20,17 @@ Or to install with support for the ``TestRPCProvider`` and
 
    $ pip install web3[tester]
 
+
+Or to install with support for gevent based threading:
+
+.. code-block:: shell
+
+   $ pip install web3[gevent]
+
+
+To enable gevent based threading set the environment variable ``THREADING_BACKEND=gevent``
+
+
 Installation from source can be done from the root of the project with the
 following command.
 

tests/admin-module/test_admin_setSolc.py

@@ -1,6 +1,6 @@
 import pytest
 
-from web3.utils.async import subprocess
+from web3.utils.compat import subprocess
 
 from web3.utils.string import force_text
 

tests/admin-module/test_admin_start_stop_methods.py

@@ -2,7 +2,7 @@ import pytest
 
 from flaky import flaky
 
-from web3.utils.async import socket
+from web3.utils.compat import socket
 
 
 def get_open_port():

tests/compat-utils/test_spawn.py

@@ -0,0 +1,38 @@
+from web3.utils.compat.compat_stdlib import (
+    spawn,
+    ThreadWithReturn,
+)
+
+
+class CustomThreadClass(ThreadWithReturn):
+    pass
+
+
+def test_spawning_simple_thread():
+    container = {
+        'success': None,
+    }
+
+    def target_fn():
+        container['success'] = True
+
+    thread = spawn(target_fn)
+    thread.join()
+
+    assert container['success'] is True
+
+
+def test_spawning_specific_thread_class():
+    container = {
+        'success': None,
+    }
+
+    def target_fn():
+        container['success'] = True
+
+    thread = spawn(target_fn, thread_class=CustomThreadClass)
+    thread.join()
+
+    assert isinstance(thread, CustomThreadClass)
+
+    assert container['success'] is True

tests/compat-utils/test_thread_with_return_value.py

@@ -1,15 +1,20 @@
-from web3.utils.async.stdlib_async import spawn
+from web3.utils.compat.compat_stdlib import (
+    spawn,
+)
 
 
-def test_spawning_simple_thread():
+def test_thread_with_return_value():
     container = {
         'success': None,
     }
 
     def target_fn():
         container['success'] = True
+        return 12345
 
     thread = spawn(target_fn)
     thread.join()
 
     assert container['success'] is True
+
+    assert thread.get() == 12345

tests/compat-utils/test_timeout_object.py

@@ -2,7 +2,7 @@ import pytest
 
 import time
 
-from web3.utils.async.stdlib_async import (
+from web3.utils.compat.compat_stdlib import (
     Timeout,
 )
 

tests/filtering/test_contract_on_event_filtering.py

@@ -1,7 +1,9 @@
 import pytest
 from flaky import flaky
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -22,10 +24,9 @@ def test_on_filter_using_get_interface(web3,
     txn_hash = emitter.transact().logNoArgs(emitter_event_ids.LogNoArguments)
     txn_receipt = wait_for_transaction(web3, txn_hash)
 
-    with async.Timeout(30) as timeout:
+    with Timeout(30) as timeout:
         while not filter.get(False):
-            async.sleep(sleep_interval())
-            timeout.check()
+            timeout.sleep(sleep_interval())
 
     log_entries = filter.get()
 
@@ -53,10 +54,9 @@ def test_on_filter_with_only_event_name(web3,
     txn_hash = emitter.transact().logNoArgs(emitter_event_ids.LogNoArguments)
     txn_receipt = wait_for_transaction(web3, txn_hash)
 
-    with async.Timeout(30) as timeout:
+    with Timeout(30) as timeout:
         while not seen_logs:
-            async.sleep(sleep_interval())
-            timeout.check()
+            timeout.sleep(sleep_interval())
 
     filter.stop_watching(30)
 
@@ -98,10 +98,9 @@ def test_on_filter_with_event_name_and_single_argument(web3,
     for txn_hash in txn_hashes:
         wait_for_transaction(web3, txn_hash)
 
-    with async.Timeout(30) as timeout:
+    with Timeout(30) as timeout:
         while len(seen_logs) < 2:
-            async.sleep(sleep_interval())
-            timeout.check()
+            timeout.sleep(sleep_interval())
 
     filter.stop_watching(30)
 
@@ -143,10 +142,9 @@ def test_on_filter_with_event_name_and_non_indexed_argument(web3,
     for txn_hash in txn_hashes:
         wait_for_transaction(web3, txn_hash)
 
-    with async.Timeout(30) as timeout:
+    with Timeout(30) as timeout:
         while not seen_logs:
-            async.sleep(sleep_interval())
-            timeout.check()
+            timeout.sleep(sleep_interval())
 
     filter.stop_watching(30)
 

tests/filtering/test_contract_past_event_filtering.py

@@ -2,7 +2,9 @@ import pytest
 import random
 from flaky import flaky
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -26,10 +28,9 @@ def test_past_events_filter_with_callback(web3,
     else:
         filter = Emitter.pastEvents('LogNoArguments', {}, seen_logs.append)
 
-    with async.Timeout(30) as timeout:
+    with Timeout(30) as timeout:
         while not seen_logs:
-            async.sleep(sleep_interval())
-            timeout.check()
+            timeout.sleep(sleep_interval())
 
     filter.stop_watching(30)
 
@@ -62,10 +63,9 @@ def test_past_events_filter_using_get_api(web3,
     else:
         filter = Emitter.pastEvents('LogNoArguments')
 
-    with async.Timeout(30) as timeout:
+    with Timeout(30) as timeout:
         while not filter.get(False):
-            async.sleep(sleep_interval())
-            timeout.check()
+            timeout.sleep(sleep_interval())
 
     log_entries = filter.get(False)
 

tests/filtering/test_filter_against_latest_blocks.py

@@ -1,7 +1,9 @@
 import random
 from flaky import flaky
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -16,10 +18,9 @@ def test_filter_against_latest_blocks(web3, sleep_interval, wait_for_block, skip
 
     wait_for_block(web3, current_block + 3)
 
-    with async.Timeout(5) as timeout:
+    with Timeout(5) as timeout:
         while len(seen_blocks) < 2:
-            async.sleep(sleep_interval())
-            timeout.check()
+            timeout.sleep(sleep_interval())
 
     txn_filter.stop_watching(3)
 

tests/filtering/test_filter_against_pending_transactions.py

@@ -1,7 +1,9 @@
 import random
 from flaky import flaky
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -29,10 +31,9 @@ def test_filter_against_pending_transactions(web3_empty,
     wait_for_transaction(web3, txn_1_hash)
     wait_for_transaction(web3, txn_2_hash)
 
-    with async.Timeout(5) as timeout:
+    with Timeout(5) as timeout:
         while not seen_txns:
-            async.sleep(random.random())
-            timeout.check()
+            timeout.sleep(random.random())
 
     txn_filter.stop_watching(30)
 

tests/filtering/test_filter_against_transaction_logs.py

@@ -1,7 +1,9 @@
 import random
 from flaky import flaky
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -23,10 +25,9 @@ def test_filter_against_log_events(web3_empty,
     for txn_hash in txn_hashes:
         wait_for_transaction(web3, txn_hash)
 
-    with async.Timeout(5) as timeout:
+    with Timeout(5) as timeout:
         while not seen_logs:
-            async.sleep(random.random())
-            timeout.check()
+            timeout.sleep(random.random())
 
     txn_filter.stop_watching(30)
 

tests/mining-module/test_miner_setExtra.py

@@ -3,7 +3,9 @@ import random
 from flaky import flaky
 
 from web3.utils.encoding import decode_hex
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -19,13 +21,12 @@ def test_miner_setExtra(web3_empty, wait_for_block):
 
     web3.miner.setExtra(new_extra_data)
 
-    with async.Timeout(60) as timeout:
+    with Timeout(60) as timeout:
         while True:
             extra_data = decode_hex(web3.eth.getBlock(web3.eth.blockNumber)['extraData'])
             if extra_data == new_extra_data:
                 break
-            async.sleep(random.random())
-            timeout.check()
+            timeout.sleep(random.random())
 
     after_extra = decode_hex(web3.eth.getBlock(web3.eth.blockNumber)['extraData'])
 

tests/mining-module/test_miner_setGasPrice.py

@@ -2,7 +2,9 @@ import random
 
 from flaky import flaky
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -16,10 +18,9 @@ def test_miner_setGasPrice(web3_empty, wait_for_block):
 
     web3.miner.setGasPrice(initial_gas_price // 2)
 
-    with async.Timeout(60) as timeout:
+    with Timeout(60) as timeout:
         while web3.eth.gasPrice == initial_gas_price:
-            async.sleep(random.random())
-            timeout.check()
+            timeout.sleep(random.random())
 
     after_gas_price = web3.eth.gasPrice
     assert after_gas_price < initial_gas_price

tests/mining-module/test_miner_start.py

@@ -2,7 +2,9 @@ import random
 
 from flaky import flaky
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -15,10 +17,9 @@ def test_miner_start(web3_empty, wait_for_miner_start):
 
     web3.miner.stop()
 
-    with async.Timeout(60) as timeout:
+    with Timeout(60) as timeout:
         while web3.eth.mining or web3.eth.hashrate:
-            async.sleep(random.random())
-            timeout.check()
+            timeout.sleep(random.random())
 
     assert not web3.eth.mining
     assert not web3.miner.hashrate

tests/mining-module/test_miner_stop.py

@@ -2,7 +2,9 @@ import random
 
 from flaky import flaky
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 @flaky(max_runs=3)
@@ -14,9 +16,9 @@ def test_miner_stop(web3_empty):
 
     web3.miner.stop()
 
-    with async.Timeout(60) as timeout:
+    with Timeout(60) as timeout:
         while web3.eth.mining or web3.eth.hashrate:
-            async.sleep(random.random())
+            timeout.sleep(random.random())
             timeout.check()
 
     assert not web3.eth.mining

tests/providers/conftest.py

@@ -9,11 +9,13 @@ from web3.providers.rpc import (
 from web3.providers.tester import (
     TestRPCProvider,
 )
-from web3.utils import async
+from web3.utils.compat import (
+    socket,
+)
 
 
 def get_open_port():
-    s = async.socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.bind(("", 0))
     s.listen(1)
     port = s.getsockname()[1]

tests/providers/test_testrpc_provider.py

@@ -5,7 +5,7 @@ from web3.providers.tester import (
     TestRPCProvider,
     is_testrpc_available,
 )
-from web3.utils.async import socket
+from web3.utils.compat import socket
 
 
 def get_open_port():

tests/shh-module/test_shh_filter.py

@@ -1,4 +1,4 @@
-from web3.utils.async import sleep
+from web3.utils.compat import sleep
 
 
 def test_shh_filter(web3, skip_if_testrpc):

tests/txpool-module/test_txpool_content.py

@@ -1,6 +1,8 @@
 import random
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 def test_txpool_content(web3_empty):
@@ -8,10 +10,9 @@ def test_txpool_content(web3_empty):
 
     web3.miner.stop()
 
-    with async.Timeout(60) as timeout:
+    with Timeout(60) as timeout:
         while web3.miner.hashrate or web3.eth.mining:
-            async.sleep(random.random())
-            timeout.check()
+            timeout.sleep(random.random())
 
     txn_1_hash = web3.eth.sendTransaction({
         'from': web3.eth.coinbase,

tests/txpool-module/test_txpool_inspect.py

@@ -1,6 +1,8 @@
 import random
 
-from web3.utils import async
+from web3.utils.compat import (
+    Timeout,
+)
 
 
 def test_txpool_inspect(web3_empty):
@@ -8,10 +10,9 @@ def test_txpool_inspect(web3_empty):
 
     web3.miner.stop()
 
-    with async.Timeout(60) as timeout:
+    with Timeout(60) as timeout:
         while web3.miner.hashrate or web3.eth.mining:
-            async.sleep(random.random())
-            timeout.check()
+            timeout.sleep(random.random())
 
     txn_1_hash = web3.eth.sendTransaction({
         'from': web3.eth.coinbase,

tox.ini

@@ -30,8 +30,7 @@ deps =
     stdlib: -r{toxinidir}/requirements-dev.txt
     gevent: -r{toxinidir}/requirements-gevent.txt
 setenv =
-    gevent: WEB3_ASYNC_GEVENT=True
-        TESTRPC_ASYNC_GEVENT=True
+    gevent: THREADING_BACKEND=gevent
 basepython =
     py27: python2.7
     py34: python3.4

web3/providers/ipc.py

@@ -10,8 +10,8 @@ try:
 except ImportError:
     JSONDecodeError = ValueError
 
-from web3.utils import async
-from web3.utils.async import (
+from web3.utils.compat import (
+    Timeout,
     threading,
     socket,
 )
@@ -86,24 +86,24 @@ class IPCProvider(JSONBaseProvider):
         try:
             with get_ipc_socket(self.ipc_path) as sock:
                 sock.sendall(request)
+                # TODO: use a BytesIO object here
                 response_raw = b""
 
-                with async.Timeout(10) as timeout:
+                with Timeout(10) as timeout:
                     while True:
-                        timeout.check()
                         try:
                             response_raw += sock.recv(4096)
                         except socket.timeout:
-                            async.sleep(0)
+                            timeout.sleep(0)
                             continue
 
                         if response_raw == b"":
-                            async.sleep(0)
+                            timeout.sleep(0)
                         else:
                             try:
                                 json.loads(force_text(response_raw))
                             except JSONDecodeError:
-                                async.sleep(0)
+                                timeout.sleep(0)
                                 continue
                             else:
                                 break

web3/providers/manager.py

@@ -22,7 +22,9 @@ from web3.utils.transactions import (
     serialize_transaction,
     add_signature_to_transaction,
 )
-from web3.utils import async
+from web3.utils.compat import (
+    spawn,
+)
 
 
 class RequestManager(object):
@@ -51,7 +53,7 @@ class RequestManager(object):
 
     def request_async(self, method, params):
         request_id = uuid.uuid4()
-        self.pending_requests[request_id] = async.spawn(
+        self.pending_requests[request_id] = spawn(
             self.request_blocking,
             method,
             params,

web3/providers/rpc.py

@@ -2,10 +2,12 @@ import logging
 
 from .base import JSONBaseProvider  # noqa: E402
 
-from web3.utils.compat import urlunparse
+from web3.utils.compat import (
+    urlunparse,
+    make_post_request,
+)
 from web3.utils.http import construct_user_agent
 from web3.utils.functional import cast_return_to_dict
-from web3.utils.async.http import make_post_request
 
 
 logger = logging.getLogger(__name__)

web3/providers/tester.py

@@ -1,6 +1,6 @@
 import logging
 
-from web3.utils.async import (
+from web3.utils.compat import (
     make_server,
     spawn,
 )

web3/utils/async/__init__.py

@@ -1,21 +0,0 @@
-import os
-
-
-if 'WEB3_ASYNC_GEVENT' in os.environ:
-    from .gevent_async import (  # noqa
-        Timeout,
-        sleep,
-        socket,
-        threading,
-        make_server,
-        GreenletThread,
-    )
-else:
-    from .stdlib_async import (  # noqa
-        Timeout,
-        sleep,
-        socket,
-        threading,
-        make_server,
-        GreenletThread,
-    )

web3/utils/async/gevent_async.py

@@ -1,24 +0,0 @@
-import gevent
-from gevent.pywsgi import (  # noqa: F401
-    WSGIServer,
-)
-from gevent import (  # noqa: F401
-    subprocess,
-    socket,
-    threading,
-)
-
-
-sleep = gevent.sleep
-spawn = gevent.spawn
-GreenletThread = gevent.Greenlet
-
-
-class Timeout(gevent.Timeout):
-    def check(self):
-        pass
-
-
-def make_server(host, port, application, *args, **kwargs):
-    server = WSGIServer((host, port), application, *args, **kwargs)
-    return server

web3/utils/async/http/__init__.py

@@ -1,11 +0,0 @@
-import os
-
-
-if 'WEB3_ASYNC_GEVENT' in os.environ:
-    from .gevent_http import (  # noqa: F401
-        make_post_request,
-    )
-else:
-    from .requests_http import (  # noqa: F401
-        make_post_request,
-    )

web3/utils/compat/__init__.py

@@ -1,4 +1,5 @@
 import sys
+import os
 
 
 if sys.version_info.major == 2:
@@ -11,3 +12,45 @@ else:
         urlparse,
         urlunparse,
     )
+
+
+def get_threading_backend():
+    if 'WEB3_THREADING_BACKEND' in os.environ:
+        return os.environ['WEB3_THREADING_BACKEND']
+    elif 'THREADING_BACKEND' in os.environ:
+        return os.environ['THREADING_BACKEND']
+    else:
+        return 'stdlib'
+
+
+THREADING_BACKEND = get_threading_backend()
+
+
+if THREADING_BACKEND == 'stdlib':
+    from .compat_stdlib import (
+        Timeout,
+        sleep,
+        socket,
+        threading,
+        make_server,
+        GreenletThread,
+        spawn,
+        subprocess,
+    )
+    from .compat_requests import (
+        make_post_request,
+    )
+elif THREADING_BACKEND == 'gevent':
+    from .compat_gevent import (  # noqa: F401
+        Timeout,
+        sleep,
+        socket,
+        threading,
+        make_server,
+        GreenletThread,
+        spawn,
+        subprocess,
+        make_post_request,
+    )
+else:
+    raise ValueError("Unsupported threading backend.  Must be one of 'gevent' or 'stdlib'")

web3/utils/compat/compat_gevent.py

@@ -1,5 +1,15 @@
 import collections
 
+import gevent
+from gevent.pywsgi import (  # noqa: F401
+    WSGIServer,
+)
+from gevent import (  # noqa: F401
+    subprocess,
+    socket,
+    threading,
+)
+
 import pylru
 
 from geventhttpclient import HTTPClient
@@ -10,6 +20,24 @@ from web3.utils.compat import urlparse
 _client_cache = pylru.lrucache(8)
 
 
+sleep = gevent.sleep
+spawn = gevent.spawn
+GreenletThread = gevent.Greenlet
+
+
+class Timeout(gevent.Timeout):
+    def check(self):
+        pass
+
+    def sleep(self, seconds):
+        gevent.sleep(seconds)
+
+
+def make_server(host, port, application, *args, **kwargs):
+    server = WSGIServer((host, port), application, *args, **kwargs)
+    return server
+
+
 def _get_client(host, port, **kwargs):
     ordered_kwargs = collections.OrderedDict(sorted(kwargs.items()))
     cache_key = '{0}:{1}:{2}'.format(

web3/utils/compat/compat_stdlib.py

@@ -69,6 +69,10 @@ class Timeout(Exception):
     def cancel(self):
         self.is_running = False
 
+    def sleep(self, seconds):
+        time.sleep(seconds)
+        self.check()
+
 
 class empty(object):
     pass
@@ -92,8 +96,8 @@ class ThreadWithReturn(threading.Thread):
             raise RuntimeError("Something went wrong.  No `_return` property was set")
 
 
-def spawn(target, *args, **kwargs):
-    thread = ThreadWithReturn(
+def spawn(target, thread_class=ThreadWithReturn, *args, **kwargs):
+    thread = thread_class(
         target=target,
         args=args,
         kwargs=kwargs,

web3/utils/filters.py

@@ -9,7 +9,7 @@ from .events import (
     construct_event_topic_set,
     construct_event_data_set,
 )
-from .async import (
+from .compat import (
     sleep,
     GreenletThread,
 )

web3/utils/transactions.py

@@ -25,17 +25,18 @@ from .address import (
 from .formatting import (
     pad_left,
 )
-from . import async
+from .compat import (
+    Timeout,
+)
 
 
 def wait_for_transaction_receipt(web3, txn_hash, timeout=120):
-    with async.Timeout(timeout) as _timeout:
+    with Timeout(timeout) as _timeout:
         while True:
             txn_receipt = web3.eth.getTransactionReceipt(txn_hash)
             if txn_receipt is not None:
                 break
-            async.sleep(random.random())
-            _timeout.check()
+            _timeout.sleep(random.random())
     return txn_receipt