From 949908da0ac9a57f37af7632f7bdb06cf0e0fe73 Mon Sep 17 00:00:00 2001 From: BJ Dierkes Date: Mon, 20 Jan 2014 17:10:28 -0600 Subject: [PATCH] Resolves Issue #219 --- ChangeLog | 7 +- cement/ext/ext_memcached.py | 218 +++++++++++++++++++++++++++ doc/source/api/ext/ext_memcached.rst | 10 ++ doc/source/api/index.rst | 9 +- requirements-dev.txt | 3 +- setup.cfg | 1 + tests/ext/memcached_tests.py | 78 ++++++++++ 7 files changed, 318 insertions(+), 8 deletions(-) create mode 100644 cement/ext/ext_memcached.py create mode 100644 doc/source/api/ext/ext_memcached.rst create mode 100644 tests/ext/memcached_tests.py diff --git a/ChangeLog b/ChangeLog index c19511c9..b32fe0b5 100755 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,7 @@ Features: * :issue:`209` - Added app.debug property to allow developers to know if `--debug` was passed at command line of via the config. + * :issue:`219` - Merged ext.memcached into mainline. 2.1.4 - Tue Oct 29, 2013 (DEVELOPMENT) @@ -43,12 +44,12 @@ Bugs: Features: - * :issue:`190` - Merge daemon extension into core + * :issue:`190` - Merged daemon extension into core * :issue:`194` - Added pre_argument_parsing/post_argument_parsing hooks * :issue:`196` - Added utils.misc.wrap - * :issue:`200` - Merge ext.mustache into mainline. + * :issue:`200` - Merged ext.mustache into mainline. * :issue:`203` - Added support for external template directory - * :issue:`207` - Support alternative 'display' name for stacked + * :issue:`207` - Added support for alternative 'display' name for stacked controllers Incompatible: diff --git a/cement/ext/ext_memcached.py b/cement/ext/ext_memcached.py new file mode 100644 index 00000000..b6705d3b --- /dev/null +++ b/cement/ext/ext_memcached.py @@ -0,0 +1,218 @@ +"""Memcached Framework Extension.""" + +import sys +import pylibmc +from ..core import cache, handler +from ..utils.misc import minimal_logger + +LOG = minimal_logger(__name__) + + +class MemcachedCacheHandler(cache.CementCacheHandler): + """ + This class implements the :ref:`ICache ` + interface. It provides a caching interface using the + `pylibmc `_ library. + + **Note** This extension has an external dependency on `pylibmc`. You + must include `pylibmc` in your applications dependencies as Cement + explicitly does *not* include external dependencies for optional + extensions. + + **Note** This extension is not supported on Python 3, due to the fact + that `pylibmc` does not appear to support Python 3 as of yet. + + Configuration: + + The Memcached extension is configurable with the following config settings + under a `[cache.memcached]` section of the application configuration. + + * **expire_time** - The default time in second to expire items in the + cache. Default: 0 (does not expire). + * **hosts** - List of Memcached servers. + + Configurations can be passed as defaults to a CementApp: + + .. code-block:: python + + from cement.core import foundation, backend + from cement.utils.misc import init_defaults + + defaults = init_defaults('myapp', 'cache.memcached') + defaults['cache.memcached']['expire_time'] = 0 + defaults['cache.memcached']['hosts'] = ['127.0.0.1'] + + app = foundation.CementApp('myapp', + config_defaults=defaults, + cache_handler='memcached', + ) + + + Additionally, an application configuration file might have a section like + the following: + + .. code-block:: text + + [cache.memcached] + + # time in seconds that an item in the cache will expire + expire_time = 3600 + + # comma seperated list of memcached servers + hosts = 127.0.0.1, cache.example.com + + + Usage: + + .. code-block:: python + + from cement.core import foundation + from cement.utils.misc import init_defaults + + defaults = init_defaults('myapp', 'memcached') + defaults['cache.memcached']['expire_time'] = 300 # seconds + defaults['cache.memcached']['hosts'] = ['127.0.0.1'] + + class MyApp(foundation.CementApp): + class Meta: + label = 'myapp' + config_defaults = defaults + extensions = ['memcached'] + cache_handler = 'memcached' + + app = MyApp() + try: + app.setup() + app.run() + + # Set a cached value + app.cache.set('my_key', 'my value') + + # Get a cached value + app.cache.get('my_key') + + # Delete a cached value + app.cache.delete('my_key') + + # Delete the entire cache + app.cache.purge() + + finally: + app.close() + + """ + class Meta: + interface = cache.ICache + label = 'memcached' + config_defaults = dict( + hosts=['127.0.0.1'], + expire_time=0, + ) + + def __init__(self, *args, **kw): + super(MemcachedCacheHandler, self).__init__(*args, **kw) + self.mc = None + + def _setup(self, *args, **kw): + super(MemcachedCacheHandler, self)._setup(*args, **kw) + self._fix_hosts() + self.mc = pylibmc.Client(self._config('hosts')) + + def _fix_hosts(self): + """ + Useful to fix up the hosts configuration (i.e. convert a + comma-separated string into a list). This function does not return + anything, however it is expected to set the `hosts` value of the + `[cache.memcached]` section (which is what this extension reads for + it's host configution). + + :returns: None + + """ + hosts = self._config('hosts') + fixed_hosts = [] + + if type(hosts) == str: + parts = hosts.split(',') + for part in parts: + fixed_hosts.append(part.strip()) + elif type(hosts) == list: + fixed_hosts = hosts + self.app.config.set(self._meta.config_section, 'hosts', fixed_hosts) + + def get(self, key, fallback=None, **kw): + """ + Get a value from the cache. Any additional keyword arguments will be + passed directly to `pylibmc` get function. + + :param key: The key of the item in the cache to get. + :param fallback: The value to return if the item is not found in the + cache. + :returns: The value of the item in the cache, or the `fallback` value. + + """ + LOG.debug("getting cache value using key '%s'" % key) + res = self.mc.get(key, **kw) + if res is None: + return fallback + else: + return res + + def _config(self, key): + """ + This is a simple wrapper, and is equivalent to: + `self.app.config.get('cache.memcached', )`. + + :param key: The key to get a config value from the 'cache.memcached' + config section. + :returns: The value of the given key. + + """ + return self.app.config.get(self._meta.config_section, key) + + def set(self, key, value, time=None, **kw): + """ + Set a value in the cache for the given `key`. Any additional + keyword arguments will be passed directly to the `pylibmc` set + function. + + :param key: The key of the item in the cache to set. + :param value: The value of the item to set. + :param time: The expiration time (in seconds) to keep the item cached. + Defaults to `expire_time` as defined in the applications + configuration. + :returns: None + + """ + if time is None: + time = int(self._config('expire_time')) + + self.mc.set(key, value, time=time, **kw) + + def delete(self, key, **kw): + """ + Delete an item from the cache for the given `key`. Any additional + keyword arguments will be passed directly to the `pylibmc` delete + function. + + :param key: The key to delete from the cache. + :returns: None + + """ + self.mc.delete(key, **kw) + + def purge(self, **kw): + """ + Purge the entire cache, all keys and values will be lost. Any + additional keyword arguments will be passed directly to the + `pylibmc` flush_all() function. + + :returns: None + + """ + + self.mc.flush_all(**kw) + + +def load(): + handler.register(MemcachedCacheHandler) diff --git a/doc/source/api/ext/ext_memcached.rst b/doc/source/api/ext/ext_memcached.rst new file mode 100644 index 00000000..8d1273f4 --- /dev/null +++ b/doc/source/api/ext/ext_memcached.rst @@ -0,0 +1,10 @@ +.. _cement.ext.ext_memcached: + +:mod:`cement.ext.ext_memcached` +------------------------------- + +.. automodule:: cement.ext.ext_memcached + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst index cf11fc30..73e2926c 100644 --- a/doc/source/api/index.rst +++ b/doc/source/api/index.rst @@ -8,7 +8,7 @@ Cement Core Modules .. toctree:: :maxdepth: 1 - + core/arg core/backend core/cache @@ -32,12 +32,12 @@ Cement Utility Modules .. toctree:: :maxdepth: 1 - + utils/fs utils/shell utils/misc utils/test - + .. _api-ext: Cement Extension Modules @@ -45,12 +45,13 @@ Cement Extension Modules .. toctree:: :maxdepth: 1 - + ext/ext_argparse ext/ext_configparser ext/ext_daemon ext/ext_json ext/ext_logging + ext/ext_memcached ext/ext_mustache ext/ext_nulloutput ext/ext_plugin diff --git a/requirements-dev.txt b/requirements-dev.txt index 0847807f..b307fa34 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,5 @@ pep8 autopep8 # Required for optional extensions -pystache \ No newline at end of file +pystache +pylibmc diff --git a/setup.cfg b/setup.cfg index 077923c1..a75b0d96 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,6 +8,7 @@ cover-inclusive=1 cover-erase=1 cover-html=1 cover-html-dir=coverage_report/ +where=tests/ [build_sphinx] source-dir = doc/source diff --git a/tests/ext/memcached_tests.py b/tests/ext/memcached_tests.py new file mode 100644 index 00000000..5c236fa0 --- /dev/null +++ b/tests/ext/memcached_tests.py @@ -0,0 +1,78 @@ +"""Tests for cement.ext.ext_memcached.""" + +import sys +from time import sleep +from random import random +from cement.core import handler +from cement.utils import test +from cement.utils.misc import init_defaults + +if sys.version_info[0] < 3: + import pylibmc +else: + raise test.SkipTest('pylibmc does not support Python 3') # pragma: no cover + +class MemcachedExtTestCase(test.CementTestCase): + def setUp(self): + self.key = "cement-tests-random-key-%s" % random() + defaults = init_defaults('tests', 'cache.memcached') + defaults['cache.memcached']['hosts'] = '127.0.0.1, localhost' + self.app = self.make_app('tests', + config_defaults=defaults, + extensions=['memcached'], + cache_handler='memcached', + ) + self.app.setup() + + def tearDown(self): + self.app.cache.delete(self.key) + + def test_memcache_list_type_config(self): + defaults = init_defaults('tests', 'cache.memcached') + defaults['cache.memcached']['hosts'] = ['127.0.0.1', 'localhost'] + self.app = self.make_app('tests', + config_defaults=defaults, + extensions=['memcached'], + cache_handler='memcached', + ) + self.app.setup() + self.eq(self.app.config.get('cache.memcached', 'hosts'), + ['127.0.0.1', 'localhost']) + + def test_memcache_str_type_config(self): + defaults = init_defaults('tests', 'cache.memcached') + defaults['cache.memcached']['hosts'] = '127.0.0.1, localhost' + self.app = self.make_app('tests', + config_defaults=defaults, + extensions=['memcached'], + cache_handler='memcached', + ) + self.app.setup() + self.eq(self.app.config.get('cache.memcached', 'hosts'), + ['127.0.0.1', 'localhost']) + + def test_memcached_set(self): + self.app.cache.set(self.key, 1001) + self.eq(self.app.cache.get(self.key), 1001) + + def test_memcached_get(self): + # get empty value + self.app.cache.delete(self.key) + self.eq(self.app.cache.get(self.key), None) + + # get empty value with fallback + self.eq(self.app.cache.get(self.key, 1234), 1234) + + def test_memcached_delete(self): + self.app.cache.delete(self.key) + + def test_memcached_purge(self): + self.app.cache.set(self.key, 1002) + self.app.cache.purge() + self.eq(self.app.cache.get(self.key), None) + + def test_memcache_expire(self): + self.app.cache.set(self.key, 1003, time=2) + sleep(3) + self.eq(self.app.cache.get(self.key), None) +