convert all tests to use parametrized web3 fixture

.travis.yml

@@ -2,6 +2,12 @@ sudo: false
 language: python
 python:
   - "3.5"
+dist: trusty
+sudo: required
+before_install:
+  - sudo add-apt-repository -y ppa:ethereum/ethereum
+  - sudo apt-get update
+  - sudo apt-get install -y ethereum
 env:
   matrix:
     - TOX_ENV=py27

conftest.py

@@ -11,16 +11,67 @@ def get_open_port():
     return port
 
 
-@pytest.yield_fixture()
-def web3_tester():
-    from web3 import Web3
-    from web3.web3.rpcprovider import TestRPCProvider
+def wait_for_http_connection(port, timeout=30):
+    import gevent
+    import requests
 
-    provider = TestRPCProvider(port=get_open_port())
+    with gevent.Timeout(timeout):
+        while True:
+            try:
+                requests.post("http://127.0.0.1:{0}".format(port))
+            except requests.ConnectionError:
+                gevent.sleep(0.1)
+                continue
+            else:
+                break
+        else:
+            raise ValueError("Unable to establish HTTP connection")
+
+
+def wait_for_ipc_connection(ipc_path, timeout=30):
+    import os
+    import gevent
+
+    with gevent.Timeout(timeout):
+        while True:
+            if os.path.exists(ipc_path):
+                break
+            gevent.sleep(0.1)
+        else:
+            raise ValueError("Unable to establish HTTP connection")
+
+
+@pytest.yield_fixture(params=['tester', 'rpc', 'ipc'])
+def web3(request, tmpdir):
+    from web3 import Web3
+    from pygeth.geth import DevGethProcess
+
+    if request.param == "tester":
+        from web3.web3.rpcprovider import TestRPCProvider
+        provider = TestRPCProvider(port=get_open_port())
+    elif request.param == "rpc":
+        from web3.web3.rpcprovider import RPCProvider
+        geth = DevGethProcess('testing', base_dir=str(tmpdir.mkdir("data-dir")))
+        geth.start()
+        wait_for_http_connection(geth.rpc_port)
+        provider = RPCProvider(port=geth.rpc_port)
+    elif request.param == "ipc":
+        from web3.web3.ipcprovider import IPCProvider
+        geth = DevGethProcess('testing', base_dir=str(tmpdir.mkdir("data-dir")))
+        geth.start()
+        wait_for_ipc_connection(geth.ipc_path)
+        provider = IPCProvider(geth.ipc_path)
+    else:
+        raise ValueError("Unknown param")
 
     _web3 = Web3(provider)
 
     yield _web3
 
-    _web3.currentProvider.server.shutdown()
-    _web3.currentProvider.server.server_close()
+    if request.param == "tester":
+        _web3.currentProvider.server.shutdown()
+        _web3.currentProvider.server.server_close()
+    elif request.param in {"rpc", "ipc"}:
+        geth.stop()
+    else:
+        raise ValueError("Unknown param")

requirements-dev.txt

@@ -1,6 +1,7 @@
 pytest>=2.8.2
 pytest-pythonpath>=0.3
 tox>=1.8.0
-eth-testrpc==0.2.1
+eth-testrpc==0.2.2
+py-geth>=0.1.0
 # Until pyethereum > 1.3.6 is released.
 https://github.com/ethereum/pyethereum/tarball/b06829e56e2a3de5276077a611ed8813b6cf5e74

tests/db-module/test_db_methods.py

@@ -1,21 +1,21 @@
 import pytest
 
 
-def test_putString(web3_tester):
+def test_putString(web3):
     with pytest.raises(DeprecationWarning):
-        web3_tester.db.putString('someDB', 'key', 'value')
+        web3.db.putString('someDB', 'key', 'value')
 
 
-def test_getString(web3_tester):
+def test_getString(web3):
     with pytest.raises(DeprecationWarning):
-        web3_tester.db.getString('someDB', 'key')
+        web3.db.getString('someDB', 'key')
 
 
-def test_putHex(web3_tester):
+def test_putHex(web3):
     with pytest.raises(DeprecationWarning):
-        web3_tester.db.putHex('someDB', 'key', '0x12345')
+        web3.db.putHex('someDB', 'key', '0x12345')
 
 
-def test_getHex(web3_tester):
+def test_getHex(web3):
     with pytest.raises(DeprecationWarning):
-        web3_tester.db.getHex('someDB', 'key')
+        web3.db.getHex('someDB', 'key')

tests/eth-module/test_eth_coinbase.py

@@ -0,0 +1,3 @@
+def test_coinbase_property(web3):
+    cb = web3.eth.coinbase
+    assert len(cb) == 42

tests/eth-module/test_getCoinbase.py

@@ -1,7 +0,0 @@
-from web3 import Web3
-from web3.web3.rpcprovider import TestRPCProvider
-
-
-def test_getCoinbase(web3_tester):
-    cb = web3_tester.eth.getCoinbase()
-    assert cb == "0x82a978b3f5962a5b0957d9ee9eef472ee55b42f1"

tests/eth-module/test_iban.py

@@ -10,8 +10,8 @@ import pytest
         ),
     )
 )
-def test_createIndirect(value, expected, web3_tester):
-    actual = web3_tester.eth.iban.createIndirect(value).toString()
+def test_createIndirect(value, expected, web3):
+    actual = web3.eth.iban.createIndirect(value).toString()
     assert actual == expected
 
 
@@ -40,8 +40,8 @@ def test_createIndirect(value, expected, web3_tester):
         ),
     ),
 )
-def test_fromAddress(value, expected, web3_tester):
-    actual = web3_tester.eth.iban.fromAddress(value).toString()
+def test_fromAddress(value, expected, web3):
+    actual = web3.eth.iban.fromAddress(value).toString()
     assert actual == expected
 
 
@@ -118,11 +118,11 @@ def test_fromAddress(value, expected, web3_tester):
         ),
     ),
 )
-def test_isValid(value, expected, web3_tester):
-    actual = web3_tester.eth.iban.isValid(value)
+def test_isValid(value, expected, web3):
+    actual = web3.eth.iban.isValid(value)
     assert actual is expected
 
-    iban = web3_tester.eth.iban(value)
+    iban = web3.eth.iban(value)
     assert iban.isValid() is expected
 
 
@@ -136,6 +136,6 @@ def test_isValid(value, expected, web3_tester):
         ),
     )
 )
-def test_toAddress(value, expected, web3_tester):
-    actual = web3_tester.eth.iban(value).address()
+def test_toAddress(value, expected, web3):
+    actual = web3.eth.iban(value).address()
     assert actual == expected

tests/net-module/test_net_properties.py

@@ -1,6 +1,6 @@
-def test_net_listening(web3_tester):
-    assert web3_tester.net.listening is False
+def test_net_listening(web3):
+    assert web3.net.listening is False
 
 
-def test_net_peerCount(web3_tester):
-    assert web3_tester.net.peerCount == 0
+def test_net_peerCount(web3):
+    assert web3.net.peerCount == 0

tests/personal-module/test_personal_module.py

@@ -0,0 +1,7 @@
+import pytest
+
+
+def test_personal_listAccounts(web3):
+    with pytest.raises(ValueError):
+        # this method is not implemented in the `eth-testrpc` server
+        web3.personal.listAccounts

tests/providers/test_tester_provider.py

@@ -1,16 +1,25 @@
 import pytest
+import socket
 
 from web3.web3.requestmanager import RequestManager
 from web3.web3.rpcprovider import TestRPCProvider, is_testrpc_available
 
 
+def get_open_port():
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.bind(("", 0))
+    s.listen(1)
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+
 @pytest.mark.skipif(not is_testrpc_available, reason="`eth-testrpc` is not installed")
 def test_making_provider_request():
     from testrpc.testrpc import web3_clientVersion
-    provider = TestRPCProvider()
+    provider = TestRPCProvider(port=get_open_port())
     rm = RequestManager(provider)
 
-    req_id = rm.forward({"method": "web3_clientVersion", "params": []})
-    response = rm.receive(req_id, timeout=1)
+    response = rm.request_blocking(method="web3_clientVersion", params=[])
 
     assert response == web3_clientVersion()

tests/version/test_web3_version_props.py

@@ -2,17 +2,17 @@ import web3
 import testrpc
 
 
-def test_api_property(web3_tester):
-    assert web3_tester.version.api == web3.__version__
+def test_api_property(web3):
+    assert web3.version.api == web3.__version__
 
 
-def test_node_property(web3_tester):
-    assert web3_tester.version.node == testrpc.testrpc.web3_clientVersion()
+def test_node_property(web3):
+    assert web3.version.node == testrpc.testrpc.web3_clientVersion()
 
 
-def test_network_property(web3_tester):
-    assert web3_tester.version.network == testrpc.testrpc.net_version()
+def test_network_property(web3):
+    assert web3.version.network == testrpc.testrpc.net_version()
 
 
-def test_ethereum_property(web3_tester):
-    assert web3_tester.version.ethereum == testrpc.testrpc.eth_protocolVersion()
+def test_ethereum_property(web3):
+    assert web3.version.ethereum == testrpc.testrpc.eth_protocolVersion()

web3/utils/encoding.py

@@ -1,4 +1,5 @@
 # String encodings and numeric representations
+import sys
 import binascii
 from . import utils
 import json
@@ -99,3 +100,105 @@ def abiToJson(obj):
     Converts abi string to python object
     """
     return json.loads(obj)
+
+
+if sys.version_info.major == 2:
+    binary_types = (bytes, bytearray)
+    text_types = (unicode,)  # NOQA
+    string_types = (basestring, bytearray)  # NOQA
+else:
+    binary_types = (bytes, bytearray)
+    text_types = (str,)
+    string_types = (bytes, str, bytearray)
+
+
+def is_binary(value):
+    return isinstance(value, binary_types)
+
+
+def is_text(value):
+    return isinstance(value, text_types)
+
+
+def is_string(value):
+    return isinstance(value, string_types)
+
+
+if sys.version_info.major == 2:
+    def force_bytes(value):
+        if is_binary(value):
+            return str(value)
+        elif is_text(value):
+            return value.encode('latin1')
+        else:
+            raise TypeError("Unsupported type: {0}".format(type(value)))
+
+    def force_text(value):
+        if is_text(value):
+            return value
+        elif is_binary(value):
+            return unicode(force_bytes(value), 'latin1')  # NOQA
+        else:
+            raise TypeError("Unsupported type: {0}".format(type(value)))
+else:
+    def force_bytes(value):
+        if is_binary(value):
+            return bytes(value)
+        elif is_text(value):
+            return bytes(value, 'latin1')
+        else:
+            raise TypeError("Unsupported type: {0}".format(type(value)))
+
+    def force_text(value):
+        if isinstance(value, text_types):
+            return value
+        elif isinstance(value, binary_types):
+            return str(value, 'latin1')
+        else:
+            raise TypeError("Unsupported type: {0}".format(type(value)))
+
+
+def force_obj_to_bytes(obj, skip_unsupported=False):
+    if is_string(obj):
+        return force_bytes(obj)
+    elif isinstance(obj, dict):
+        return {
+            k: force_obj_to_bytes(v, skip_unsupported) for k, v in obj.items()
+        }
+    elif isinstance(obj, (list, tuple)):
+        return type(obj)(force_obj_to_bytes(v, skip_unsupported) for v in obj)
+    elif not skip_unsupported:
+        raise ValueError("Unsupported type: {0}".format(type(obj)))
+    else:
+        return obj
+
+
+def force_obj_to_text(obj, skip_unsupported=False):
+    if is_string(obj):
+        return force_text(obj)
+    elif isinstance(obj, dict):
+        return {
+            k: force_obj_to_text(v, skip_unsupported) for k, v in obj.items()
+        }
+    elif isinstance(obj, (list, tuple)):
+        return type(obj)(force_obj_to_text(v, skip_unsupported) for v in obj)
+    elif not skip_unsupported:
+        raise ValueError("Unsupported type: {0}".format(type(obj)))
+    else:
+        return obj
+
+
+def coerce_args_to_bytes(fn):
+    @functools.wraps(fn)
+    def inner(*args, **kwargs):
+        bytes_args = force_obj_to_bytes(args, True)
+        bytes_kwargs = force_obj_to_bytes(kwargs, True)
+        return fn(*bytes_args, **bytes_kwargs)
+    return inner
+
+
+def coerce_return_to_bytes(fn):
+    @functools.wraps(fn)
+    def inner(*args, **kwargs):
+        return force_obj_to_bytes(fn(*args, **kwargs), True)
+    return inner

web3/web3/methods/eth.py

@@ -274,13 +274,19 @@ class Eth(object):
     def getBlockNumber(self, *args, **kwargs):
         raise NotImplementedError("Async calling has not been implemented")
 
-    def getBalance(self, account, block_number="latest"):
+    def getBalance(self, account, block_number=None):
+        if block_number is None:
+            block_number = self.defaultBlock
         return self.request_manager.request_blocking("eth_getBalance", [account, block_number])
 
-    def getStorageAt(self, account, position, block_number="latest")
+    def getStorageAt(self, account, position, block_number=None):
+        if block_number is None:
+            block_number = self.defaultBlock
         return self.request_manager.request_blocking("eth_getStorageAt", [account, position, block_number])
 
-    def getCode(self, account, block_number="latest")
+    def getCode(self, account, block_number=None):
+        if block_number is None:
+            block_number = self.defaultBlock
         return self.request_manager.request_blocking("eth_getCode", [account, block_number])
 
     def getBlock(self, block_identifier, full_txns):
@@ -313,7 +319,9 @@ class Eth(object):
     def getTransactionReciept(self, txn_hash):
         return self.request_manager.request_blocking("eth_getTransactionReceipt", [txn_hash])
 
-    def getTransactionCount(self, account, block_number="latest"):
+    def getTransactionCount(self, account, block_number=None):
+        if block_number is None:
+            block_number = self.defaultBlock
         return self.request_manager.request_blocking("eth_getTransactionCount", [account, block_number])
 
     def sendTransaction(self, *args, **kwargs):

web3/web3/methods/personal.py

@@ -1,29 +1,3 @@
-import web3.web3.formatters as formatters
-
-
-# TODO: remove this list
-methods = [
-    {
-        "name": "newAccount",
-        "call": "personal_newAccount",
-        "params": 1,
-        "inputFormatter": [None]
-    },
-    {
-        "name": "unlockAccount",
-        "call": "personal_unlockAccount",
-        "params": 3,
-        "inputFormatter": [formatters.inputAddressFormatter, None, None]
-    },
-    {
-        "name": "lockAccount",
-        "call": "personal_lockAccount",
-        "params": 1,
-        "inputFormatter": [formatters.inputAddressFormatter]
-    }
-]
-
-
 class Personal(object):
     """
     https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal

web3/web3/provider.py

@@ -1,6 +1,10 @@
+from __future__ import absolute_import
+
 import json
 import itertools
 
+from web3.utils.encoding import force_bytes
+
 
 class BaseProvider(object):
     def __init__(self):
@@ -10,9 +14,9 @@ class BaseProvider(object):
         raise NotImplementedError("Providers must implement this method")
 
     def encode_rpc_request(self, method, params):
-        return json.dumps({
+        return force_bytes(json.dumps({
             "jsonrpc": "2.0",
             "method": method,
             "params": params or [],
             "id": next(self.request_counter),
-        })
+        }))

web3/web3/requestmanager.py

@@ -2,6 +2,8 @@ import uuid
 import json
 import gevent
 
+from web3.utils.encoding import force_text
+
 
 class RequestManager(object):
     def __init__(self, provider):
@@ -14,7 +16,7 @@ class RequestManager(object):
         """
         response_raw = self.provider.make_request(method, params)
 
-        response = json.loads(response_raw)
+        response = json.loads(force_text(response_raw))
 
         if "error" in response:
             raise ValueError(response["error"])