8000 Stop listing if recursion limit is hit (#234) · jflemer-ndp/python-gitlab@989f3b7 · GitHub
[go: up one dir, main page]
Skip to content

Commit 989f3b7

Browse files
Johan BrandhorstGauvain Pocentek
Johan Brandhorst
authored and
Gauvain Pocentek
committed
Stop listing if recursion limit is hit (python-gitlab#234)
1 parent 22bf128 commit 989f3b7

File tree

3 files changed

+93
-11
lines changed
  • gitlab

    • 3 files changed

    +93
    -11
    lines changed

    ‎docs/api-usage.rst

    Lines changed: 3 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -142,7 +142,9 @@ parameter to get all the items when using listing methods:
    142142
    python-gitlab will iterate over the list by calling the correspnding API
    143143
    multiple times. This might take some time if you have a lot of items to
    144144
    retrieve. This might also consume a lot of memory as all the items will be
    145-
    stored in RAM.
    145+
    stored in RAM. If you're encountering the python recursion limit exception,
    146+
    use ``safe_all=True`` instead to stop pagination automatically if the
    147+
    recursion limit is hit.
    146148

    147149
    Sudo
    148150
    ====

    ‎gitlab/__init__.py

    Lines changed: 20 additions & 10 deletions
    Original file line numberDiff line numberDiff line change
    @@ -112,8 +112,8 @@ def __init__(self, url, private_token=None, email=None, password=None,
    112112
    # build the "submanagers"
    113113
    for parent_cls in six.itervalues(globals()):
    114114
    if (not inspect.isclass(parent_cls)
    115-
    or not issubclass(parent_cls, GitlabObject)
    116-
    or parent_cls == CurrentUser):
    115+
    or not issubclass(parent_cls, GitlabObject)
    116+
    or parent_cls == CurrentUser):
    117117
    continue
    118118

    119119
    if not parent_cls.managers:
    @@ -312,11 +312,13 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs):
    312312
    params = extra_attrs.copy()
    313313
    params.update(kwargs.copy())
    314314

    315-
    get_all_results = kwargs.get('all', False)
    315+
    catch_recursion_limit = kwargs.get('safe_all', False)
    316+
    get_all_results = (kwargs.get('all', False) is True
    317+
    or catch_recursion_limit)
    316318

    317319
    # Remove these keys to avoid breaking the listing (urls will get too
    318320
    # long otherwise)
    319-
    for key in ['all', 'next_url']:
    321+
    for key in ['all', 'next_url', 'safe_all']:
    320322
    if key in params:
    321323
    del params[key]
    322324

    @@ -334,12 +336,20 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs):
    334336

    335337
    results = [cls(self, item, **params) for item in r.json()
    336338
    if item is not None]
    337-
    if ('next' in r.links and 'url' in r.links['next']
    338-
    and get_all_results is True):
    339-
    args = kwargs.copy()
    340-
    args.update(extra_attrs)
    341-
    args['next_url'] = r.links['next']['url']
    342-
    results.extend(self.list(cls, **args))
    339+
    try:
    340+
    if ('next' in r.links and 'url' in r.links['next']
    341+
    and get_all_results):
    342+
    args = kwargs.copy()
    343+
    args.update(extra_attrs)
    344+
    args['next_url'] = r.links['next']['url']
    345+
    results.extend(self.list(cls, **args))
    346+
    except Exception as e:
    347+
    # Catch the recursion limit exception if the 'safe_all'
    348+
    # kwarg was provided
    349+
    if not (catch_recursion_limit and
    350+
    "maximum recursion depth exceeded" in str(e)):
    351+
    raise e
    352+
    343353
    return results
    344354

    345355
    def _raw_post(self, path_, data=None, content_type=None, **kwargs):

    ‎gitlab/tests/test_gitlab.py

    Lines changed: 70 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -26,6 +26,7 @@
    2626
    from httmock import HTTMock # noqa
    2727
    from httmock import response # noqa
    2828
    from httmock import urlmatch # noqa
    29+
    import six
    2930

    3031
    import gitlab
    3132
    from gitlab import * # noqa
    @@ -243,6 +244,75 @@ def resp_two(url, request):
    243244
    self.assertEqual(data[0].ref, "b")
    244245
    self.assertEqual(len(data), 2)
    245246

    247+
    def test_list_recursion_limit_caught(self):
    248+
    @urlmatch(scheme="http", netloc="localhost",
    249+
    path='/api/v3/projects/1/repository/branches', method="get")
    250+
    def resp_one(url, request):
    251+
    """First request:
    252+
    253+
    http://localhost/api/v3/projects/1/repository/branches?per_page=1
    254+
    """
    255+
    headers = {
    256+
    'content-type': 'application/json',
    257+
    'link': '<http://localhost/api/v3/projects/1/repository/branc'
    258+
    'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3'
    259+
    '/projects/1/repository/branches?page=2&per_page=0>; rel="las'
    260+
    't", <http://localhost/api/v3/projects/1/repository/branches?'
    261+
    'page=1&per_page=0>; rel="first"'
    262+
    }
    263+
    content = ('[{"branch_name": "otherbranch", '
    264+
    '"project_id": 1, "ref": "b"}]').encode("utf-8")
    265+
    resp = response(200, content, headers, None, 5, request)
    266+
    return resp
    267+
    268+
    @urlmatch(scheme="http", netloc="localhost",
    269+
    path='/api/v3/projects/1/repository/branches', method="get",
    270+
    query=r'.*page=2.*')
    271+
    def resp_two(url, request):
    272+
    # Mock a runtime error
    273+
    raise RuntimeError("maximum recursion depth exceeded")
    274+
    275+
    with HTTMock(resp_two, resp_one):
    276+
    data = self.gl.list(ProjectBranch, project_id=1, per_page=1,
    277+
    safe_all=True)
    278+
    self.assertEqual(data[0].branch_name, "otherbranch")
    279+
    self.assertEqual(data[0].project_id, 1)
    280+
    self.assertEqual(data[0].ref, "b")
    281+
    self.assertEqual(len(data), 1)
    282+
    283+
    def test_list_recursion_limit_not_caught(self):
    284+
    @urlmatch(scheme="http", netloc="localhost",
    285+
    path='/api/v3/projects/1/repository/branches', method="get")
    286+
    def resp_one(url, request):
    287+
    """First request:
    288+
    289+
    http://localhost/api/v3/projects/1/repository/branches?per_page=1
    290+
    """
    291+
    headers = {
    292+
    'content-type': 'application/json',
    293+
    'link': '<http://localhost/api/v3/projects/1/repository/branc'
    294+
    'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3'
    295+
    '/projects/1/repository/branches?page=2&per_page=0>; rel="las'
    296+
    't", <http://localhost/api/v3/projects/1/repository/branches?'
    297+
    'page=1&per_page=0>; rel="first"'
    298+
    }
    299+
    content = ('[{"branch_name": "otherbranch", '
    300+
    '"project_id": 1, "ref": "b"}]').encode("utf-8")
    301+
    resp = response(200, content, headers, None, 5, request)
    302+
    return resp
    303+
    304+
    @urlmatch(scheme="http", netloc="localhost",
    305+
    path='/api/v3/projects/1/repository/branches', method="get",
    306+
    query=r'.*page=2.*')
    307+
    def resp_two(url, request):
    308+
    # Mock a runtime error
    309+
    raise RuntimeError("maximum recursion depth exceeded")
    310+
    311+
    with HTTMock(resp_two, resp_one):
    312+
    with six.assertRaisesRegex(self, GitlabError,
    313+
    "(maximum recursion depth exceeded)"):
    314+
    self.gl.list(ProjectBranch, project_id=1, per_page=1, all=True)
    315+
    246316
    def test_list_401(self):
    247317
    @urlmatch(scheme="http", netloc="localhost",
    248318
    path="/api/v3/projects/1/repository/branches", method="get")

    0 commit comments

    Comments
     (0)
    0