Check-in [b69ee71d75]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Incompatible change by my misunderstanding. It fixed by twisted deferred design. Then it released as 0.1.
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:b69ee71d753c966feed69bc80bc2f4f94df713f5
User & Date: yusuke 2010-10-25 20:26:54
Context
2010-10-25
21:40
Updated the document for pypi recognizing of ReST. check-in: 2df0b23b38 user: yusuke tags: trunk, 0.1
20:26
Incompatible change by my misunderstanding. It fixed by twisted deferred design. Then it released as 0.1. check-in: b69ee71d75 user: yusuke tags: trunk
18:41
Added the methods callback, errback and chainDeferred. And it's a twisted standard I should follow. check-in: 9de48dcace user: yusuke tags: trunk
Changes

Changes to README.txt.

35
36
37
38
39
40
41
42





43
44
45
46
47
48
49







50
51
52
53
54
55
56
57
58
59
60
61
62
    >>> deferred.defer(d.callback, 5)

How to test
-----------

Like this.:: bash

  $ py.test --appengine=${your appengine path} --doctest-glob='*.rst'






Dependencies
------------

* py.test
  It is used for testing.








To do
-----

* deferred list

Questions and Bug Reports
-------------------------

* Site
  https://chiselapp.com/user/jbking/repository/gaedeferred/

* Or, cantact me.
  MURAOKA Yusuke <yusuke.muraoka@gmail.com>







|
>
>
>
>
>







>
>
>
>
>
>
>













35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
    >>> deferred.defer(d.callback, 5)

How to test
-----------

Like this.:: bash

  $ py.test --appengine=${your appengine path} --doctest-glob='*.txt'

Or:: bash

  $ export APPENGINE_PATH=${your appengine path}
  $ py.test --doctest-glob='*.txt'

Dependencies
------------

* py.test
  It is used for testing.

Changelog
---------

* 26-Oct-2010
  Incompatible API change between 0.1dev to 0.1.
  It was my misunderstanding of Twisted deferred.

To do
-----

* deferred list

Questions and Bug Reports
-------------------------

* Site
  https://chiselapp.com/user/jbking/repository/gaedeferred/

* Or, cantact me.
  MURAOKA Yusuke <yusuke.muraoka@gmail.com>

Changes to gaedeferred/__init__.py.

21
22
23
24
25
26
27
28

29

30
31
32
33





34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
..
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
...
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import logging
from sys import exc_info
from pickle import loads
from google.appengine.ext import deferred



class Failure(deferred.Error):
    def __init__(self, exc_value=None, exc_type=None):
        self.value = exc_value
        self.type = exc_type






def run_with_result(data, result):
    """Unpickles and executes a task with the result that is returned from previous function.
    Args:
        data: A pickled tuple of (function, args, kwargs) to execute.
        result: The returned value of previous function.
    Returns:
        The return value of the function invocation.
    """

    try:
        func, args, kwargs = loads(data)
    except Exception, e:
        raise deferred.PermanentTaskFailure(e)
    # insert the result at appropriate place.
    if func == deferred.invoke_member:
        args = args[0:2] + (result,) + args[2:]
    else:
        args = (result,) + args
    return func(*args, **kwargs)

class Deferred(object):
    def __init__(self, *args, **kwargs):
        if args:
            obj, args = args[0], args[1:]
            self.startup = deferred.serialize(obj, *args, **kwargs)
        else:
            self.startup = None

        self.callback_pairs = []

    def _callback_chain(self):
        while self.callback_pairs:
            callback, errback = self.callback_pairs.pop(0)
            if self._in_callback:
                if callback:
                    yield callback
            else:
                if errback:
                    yield errback

    def __call__(self):
        try:
            result = deferred.run(self.startup)
            logging.debug("The result of the startup execution is %s", result)
            self._in_callback = True
        except deferred.PermanentTaskFailure:
            raise
        except Exception:
            result = Failure(*exc_info()[:2])
            logging.debug("An exception occurs in the startup execution.", exc_info=True)
            self._in_callback = False
        self._callback(result)

    def callback(self, result):
        self._in_callback = True
        self._callback(result)

    def errback(self, failure):
        self._in_callback = False
        self._callback(failure)
................................................................................
        for callback in self._callback_chain():
            try:
                result = run_with_result(callback, result)
                logging.debug("The result of a callback is %s", result)
                self._in_callback = True
                if isinstance(result, Deferred):
                    result.callback_pairs.extend(self.callback_pairs)
                    # assign new task the result deferred.
                    if result.startup:
                        deferred.defer(result)
                    else:
                        deferred.defer(result.callback, None)
            except deferred.PermanentTaskFailure:
                raise
            except Exception:
                result = Failure(*exc_info()[:2])
                logging.debug("An exception occurs in callback.", exc_info=True)
                self._in_callback = False

    def addCallback(self, callback, *args, **kwargs):
................................................................................
                                 errbackArgs=args,
                                 errbackKeywords=kwargs)

    def addCallbacks(self, callback=None, callbackArgs=(), callbackKeywords={},
                     errback=None, errbackArgs=(), errbackKeywords={}):
        callback_data = None
        if callback:
            callback_data = deferred.serialize(callback,
                                               *callbackArgs,
                                               **callbackKeywords)
        errback_data = None
        if errback:
            errback_data = deferred.serialize(errback,
                                              *errbackArgs,
                                              **errbackKeywords)
        self.callback_pairs.append((callback_data, errback_data))

__all__ = ['Deferred']







|
>
|
>
|
|
<

>
>
>
>
>













|

|






|
|
|
|
|
|
>












<
<
<
<
<
<
<
<
<
<
<
<
<







 







|
|
|
<
<
<







 







|
|
|


|
|
|



21
22
23
24
25
26
27
28
29
30
31
32
33

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80













81
82
83
84
85
86
87
..
93
94
95
96
97
98
99
100
101
102



103
104
105
106
107
108
109
...
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import logging
from sys import exc_info
from pickle import loads
from google.appengine.ext.deferred import (
    Error, PermanentTaskFailure, serialize, invoke_member
)

class Failure(Error):
    def __init__(self, exc_type=None, exc_value=None):

        self.type = exc_type
        self.value = exc_value

    def trap(self, *exc_types):
        if not isinstance(self.exc_value, exc_types):
            raise self.exc_value

def run_with_result(data, result):
    """Unpickles and executes a task with the result that is returned from previous function.
    Args:
        data: A pickled tuple of (function, args, kwargs) to execute.
        result: The returned value of previous function.
    Returns:
        The return value of the function invocation.
    """

    try:
        func, args, kwargs = loads(data)
    except Exception, e:
        raise PermanentTaskFailure(e)
    # insert the result at appropriate place.
    if func == invoke_member:
        args = args[0:2] + (result,) + args[2:]
    else:
        args = (result,) + args
    return func(*args, **kwargs)

class Deferred(object):
    def __init__(self, _raise_exception_types=(PermanentTaskFailure,
                                               TypeError,)):
        """
        Args:
            _raise_exception_types: for debug use, when a deferred got an exception, and the exception is an instance of the exception type of the tuple, raise it instead of calling errback.
        """
        self._raise_exception_types = _raise_exception_types
        self.callback_pairs = []

    def _callback_chain(self):
        while self.callback_pairs:
            callback, errback = self.callback_pairs.pop(0)
            if self._in_callback:
                if callback:
                    yield callback
            else:
                if errback:
                    yield errback














    def callback(self, result):
        self._in_callback = True
        self._callback(result)

    def errback(self, failure):
        self._in_callback = False
        self._callback(failure)
................................................................................
        for callback in self._callback_chain():
            try:
                result = run_with_result(callback, result)
                logging.debug("The result of a callback is %s", result)
                self._in_callback = True
                if isinstance(result, Deferred):
                    result.callback_pairs.extend(self.callback_pairs)
                    if isinstance(result, Deferred):
                        result = None
            except self._raise_exception_types:



                raise
            except Exception:
                result = Failure(*exc_info()[:2])
                logging.debug("An exception occurs in callback.", exc_info=True)
                self._in_callback = False

    def addCallback(self, callback, *args, **kwargs):
................................................................................
                                 errbackArgs=args,
                                 errbackKeywords=kwargs)

    def addCallbacks(self, callback=None, callbackArgs=(), callbackKeywords={},
                     errback=None, errbackArgs=(), errbackKeywords={}):
        callback_data = None
        if callback:
            callback_data = serialize(callback,
                                      *callbackArgs,
                                      **callbackKeywords)
        errback_data = None
        if errback:
            errback_data = serialize(errback,
                                     *errbackArgs,
                                     **errbackKeywords)
        self.callback_pairs.append((callback_data, errback_data))

__all__ = ['Deferred']

Changes to setup.cfg.

1
2
3
[egg_info]
tag_build = dev
tag_svn_revision = true


<
1
2

[egg_info]
tag_build = dev

Changes to setup.py.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
from setuptools import setup, find_packages

version = '0.1'

setup(name='gaedeferred',
      version=version,
      description="A deferred library on taskqueue for Google Appengine",
      long_description=file("README.rst").read(),
      classifiers=["Development Status :: 3 - Alpha",
                   "Environment :: Web Environment",
                   "Intended Audience :: Developers",
                   "License :: OSI Approved :: MIT License",
                   "Programming Language :: Python :: 2.5",
                   "Topic :: Software Development :: Libraries :: Python Modules"
                  ],







|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
from setuptools import setup, find_packages

version = '0.1'

setup(name='gaedeferred',
      version=version,
      description="A deferred library on taskqueue for Google Appengine",
      long_description=file("README.txt").read(),
      classifiers=["Development Status :: 3 - Alpha",
                   "Environment :: Web Environment",
                   "Intended Audience :: Developers",
                   "License :: OSI Approved :: MIT License",
                   "Programming Language :: Python :: 2.5",
                   "Topic :: Software Development :: Libraries :: Python Modules"
                  ],

Changes to tests/conftest.py.

18
19
20
21
22
23
24
25

26
27
28
29
30
31
32
33
34
35
36
37


38
39
40
41
42
43
44
..
52
53
54
55
56
57
58







FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from os import path, sys

sys.path.insert(0, path.join(*path.split(path.abspath(path.dirname(__file__)))[:-1]))

import py

def mimic_defer(obj, *args, **kwargs):
    obj(*args, **kwargs)

def pytest_addoption(parser):
    parser.addoption("--appengine", help="The path to google appengine runtime")

def pytest_configure(config):
    runtime_path = config.getvalue("appengine") 


    if runtime_path:
        sys.path.insert(0, path.join(runtime_path, "lib/yaml/lib"))
        sys.path.insert(0, path.join(runtime_path, "lib/webob"))
        sys.path.insert(0, runtime_path)

def pytest_runtest_setup(item):
    try:
................................................................................

class Context(object):
    def __init__(self, request):
        self.config = request.config

    def defer(self, d, *args, **kwargs):
        mimic_defer(d, *args, **kwargs)














|
>








|



>
>







 







>
>
>
>
>
>
>
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
..
55
56
57
58
59
60
61
62
63
64
65
66
67
68
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from os import path, environ
import sys
sys.path.insert(0, path.join(*path.split(path.abspath(path.dirname(__file__)))[:-1]))

import py

def mimic_defer(obj, *args, **kwargs):
    obj(*args, **kwargs)

def pytest_addoption(parser):
    parser.addoption("--appengine", help="The path to google appengine runtime. Otherwise refer to the environment variable APPENGINE_PATH")

def pytest_configure(config):
    runtime_path = config.getvalue("appengine") 
    if not runtime_path:
        runtime_path = environ.get('APPENGINE_PATH')
    if runtime_path:
        sys.path.insert(0, path.join(runtime_path, "lib/yaml/lib"))
        sys.path.insert(0, path.join(runtime_path, "lib/webob"))
        sys.path.insert(0, runtime_path)

def pytest_runtest_setup(item):
    try:
................................................................................

class Context(object):
    def __init__(self, request):
        self.config = request.config

    def defer(self, d, *args, **kwargs):
        mimic_defer(d, *args, **kwargs)

    def deferred(self):
        from gaedeferred import (Deferred, PermanentTaskFailure)
        return Deferred(_raise_exception_types=(
            PermanentTaskFailure,
            TypeError,
            AssertionError))

Changes to tests/test_deferred.py.

20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

77
78
79
80
81
82
83
84

85
86
87
88
89
90
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from gaedeferred import Deferred, Failure

def value(result):
    return result

def increase(result, value=1):
    return result + value

def raise_(result):
    raise Exception("This exception occured from raise_.")

def must_be_failure(result):
    assert isinstance(result, Failure)
    assert result.value == "This exception occured from raise_."
    assert result.type == Exception

def must_be(result, value):
    assert result == value

class Object(object):
    def multiple(self, result, value=2):
        return result * value

def test_deferred(context):
    d = Deferred(value, 0)
    d.addCallback(increase)
    d.addBoth(increase, 2)
    d.addCallbacks(callback=increase, callbackArgs=(3,))
    d.next(must_be, 5) # test for the alias

    d.next(Object().multiple)
    d.next(must_be, 10)
    d.next(raise_)
    d.addErrback(must_be_failure)
    d.next(raise_)
    d.error(must_be_failure) # test for the alias
    context.defer(d)

    d = Deferred()
    d.next(must_be, 5)
    context.defer(d.callback, 5)

    d = Deferred()
    d.error(must_be, 10)
    context.defer(d.errback, 10)

def test_chain(context):
    d1 = Deferred()
    d2 = Deferred()
    d1.next(increase, 5)
    # you don't need to assing ext.defer deferred
    # a deferred treats it's next callback when you return a deferred in callback.
    d1.next(value, d2)

    d2.next(increase, 3)
    d2.next(must_be, 13)
    d1.next(must_be, 13)
    context.defer(d1.callback, 5)

    d1 = Deferred()
    d2 = Deferred()
    d1.chainDeferred(d2)

    d1.next(increase, 5)
    d2.next(increase, 3)
    d2.next(must_be, 13)
    d1.next(must_be, 13)
    context.defer(d1.callback, 5)








|
|









|










|



|
>

|




|

|



|




|
|

<
<

>


|


|
|

>



|


20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74


75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from gaedeferred import Deferred, Failure

def value(result, value):
    return value

def increase(result, value=1):
    return result + value

def raise_(result):
    raise Exception("This exception occured from raise_.")

def must_be_failure(result):
    assert isinstance(result, Failure)
    assert result.value.message == "This exception occured from raise_."
    assert result.type == Exception

def must_be(result, value):
    assert result == value

class Object(object):
    def multiple(self, result, value=2):
        return result * value

def test_deferred(context):
    d = context.deferred()
    d.addCallback(increase)
    d.addBoth(increase, 2)
    d.addCallbacks(callback=increase, callbackArgs=(3,))
    d.next(must_be, 6) # test for the alias
    d.next(value, 6)
    d.next(Object().multiple)
    d.next(must_be, 12)
    d.next(raise_)
    d.addErrback(must_be_failure)
    d.next(raise_)
    d.error(must_be_failure) # test for the alias
    context.defer(d.callback, 0)

    d = context.deferred()
    d.next(must_be, 5)
    context.defer(d.callback, 5)

    d = context.deferred()
    d.error(must_be, 10)
    context.defer(d.errback, 10)

def test_chain(context):
    d1 = context.deferred()
    d2 = context.deferred()
    d1.next(increase, 5)


    d1.next(value, d2)
    # d2 is meanless in this chain
    d2.next(increase, 3)
    d2.next(must_be, 13)
    d1.next(must_be, None)
    context.defer(d1.callback, 5)

    d1 = context.deferred()
    d2 = context.deferred()
    d1.chainDeferred(d2)
    d1.next(value, 5)
    d1.next(increase, 5)
    d2.next(increase, 3)
    d2.next(must_be, 13)
    d1.next(must_be, 10)
    context.defer(d1.callback, 5)