8000 feat: emit a warning when using a `list()` method returns max · python-gitlab/python-gitlab@92da850 · GitHub
[go: up one dir, main page]
Skip to content

Commit 92da850

Browse files
JohnVillalovosJohnVillalovos
committed
feat: emit a warning when using a list() method returns max
A common cause of issues filed and questions raised is that a user will call a `list()` method and only get 20 items. As this is the default maximum of items that will be returned from a `list()` method. To help with this we now emit a warning when the result from a `list()` method is greater-than or equal to 20 (or the specified `per_page` value) and the user is not using either `all=True`, `as_list=False`, or `page=X`.
1 parent 59df432 commit 92da850

File tree

3 files changed

+98
-6
lines changed

3 files changed

+98
-6
lines changed

‎gitlab/client.py

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@
1818

1919
import os
2020
import time
21+
import warnings
2122
from typing import Any, cast, Dict, List, Optional, Tuple, TYPE_CHECKING, Union
2223

2324
import requests
2425
import requests.utils
2526
from requests_toolbelt.multipart.encoder import MultipartEncoder # type: ignore
2627

28+
import gitlab
2729
import gitlab.config
2830
import gitlab.const
2931
import gitlab.exceptions
@@ -35,6 +37,12 @@
3537
"{source!r} to {target!r}"
3638
)
3739

40+
# https://docs.gitlab.com/ee/api/#offset-based-pagination
41+
_PAGINATION_URL = (
42+
f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
43+
f"api-usage.html#pagination"
44+
)
45+
3846

3947
class Gitlab:
4048
"""Represents a GitLab server connection.
@@ -818,7 +826,26 @@ def http_list(
818826

819827
if page or as_list is True:
820828
# pagination requested, we return a list
821-
return list(GitlabList(self, url, query_data, get_next=False, **kwargs))
829+
gl_list = GitlabList(self, url, query_data, get_next=False, **kwargs)
830+
items = list(gl_list)
831+
if (
832+
page is None
833+
and (gl_list.per_page is not None and len(items) >= gl_list.per_page)
834+
and (gl_list.total is None or len(items) < gl_list.total)
835+
):
836+
total_items = "10,000+" if gl_list.total is None else gl_list.total
837+
# Warn the user that they are only going to retrieve `per_page` maximum
838+
# items. This is a common cause of issues filed.
839+
warnings.warn(
840+
(
841+
f"Calling a `list()` method without specifying `all=True` or "
842+
f"`as_list=False` will return a maximum of {gl_list.per_page} "
843+
f"items. Your query returned {len(items)} of {total_items} "
844+
f"items. See {_PAGINATION_URL} for more details"
845+
),
846+
UserWarning,
847+
)
848+
return items
822849

823850
# No pagination, generator requested
824851
return GitlabList(self, url, query_data, **kwargs)

‎tests/functional/api/test_gitlab.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,13 @@ def test_template_dockerfile(gl):
8181

8282

8383
def test_template_gitignore(gl):
84-
assert gl.gitignores.list()
84+
assert gl.gitignores.list(all=True)
8585
gitignore = gl.gitignores.get("Node")
8686
assert gitignore.content is not None
8787

8888

8989
def test_template_gitlabciyml(gl):
90-
assert gl.gitlabciymls.list()
90+
assert gl.gitlabciymls.list(all=True)
9191
gitlabciyml = gl.gitlabciymls.get("Nodejs")
9292
assert gitlabciyml.content is not None
9393

‎tests/unit/test_gitlab_http_methods.py

Lines changed: 68 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import warnings
2+
13
import pytest
24
import requests
35
import responses
@@ -324,25 +326,88 @@ def test_list_request(gl):
324326
method=responses.GET,
325327
url=url,
326328
json=[{"name": "project1"}],
327-
headers={"X-Total": "1"},
329+
headers={"X-Total": "1", "x-per-page": "20"},
328330
status=200,
329331
match=MATCH_EMPTY_QUERY_PARAMS,
330332
)
331333

332-
result = gl.http_list("/projects", as_list=True)
334+
with warnings.catch_warnings(record=True) as caught_warnings:
335+
result = gl.http_list("/projects", as_list=True)
336+
assert len(caught_warnings) == 0
333337
assert isinstance(result, list)
334338
assert len(result) == 1
335339

336340
result = gl.http_list("/projects", as_list=False)
337341
assert isinstance(result, GitlabList)
338-
assert len(result) == 1
342+
assert len(list(result)) == 1
339343

340344
result = gl.http_list("/projects", all=True)
341345
assert isinstance(result, list)
342346
assert len(result) == 1
343347
assert responses.assert_call_count(url, 3) is True
344348

345349

350+
@responses.activate
351+
def test_list_request_pagination_warning(gl):
352+
url = "http://localhost/api/v4/projects"
353+
responses.add(
354+
method=responses.GET,
355+
url=url,
356+
json=[
357+
{"name": "project01"},
358+
{"name": "project02"},
359+
{"name": "project03"},
360+
{"name": "project04"},
361+
{"name": "project05"},
362+
{"name": "project06"},
363+
{"name": "project07"},
364+
{"name": "project08"},
365+
{"name": "project09"},
366+
{"name": "project10"},
367+
{"name": "project11"},
368+
{"name": "project12"},
369+
{"name": "project13"},
370+
{"name": "project14"},
371+
{"name": "project15"},
372+
{"name": "project16"},
373+
{"name": "project17"},
374+
{"name": "project18"},
375+
{"name": "project19"},
376+
{"name": "project20"},
377+
],
378+
headers={"X-Total": "30", "x-per-page": "20"},
379+
status=200,
380+
match=MATCH_EMPTY_QUERY_PARAMS,
381+
)
382+
383+
with warnings.catch_warnings(record=True) as caught_warnings:
384+
result = gl.http_list("/projects", as_list=True)
385+
assert len(caught_warnings) == 1
386+
warning = caught_warnings[0]
387+
assert isinstance(warning.message, UserWarning)
388+
message = str(caught_warnings[0].message)
389+
assert "Calling" in message
390+
assert "return a maximum of" in message
391+
assert "readthedocs" in message
392+
393+
assert isinstance(result, list)
394+
assert len(result) == 20
395+
396+
with warnings.catch_warnings(record=True) as caught_warnings:
397+
result = gl.http_list("/projects", as_list=False)
398+
assert len(caught_warnings) == 0
399+
assert isinstance(result, GitlabList)
400+
assert len(list(result)) == 20
401+
402+
with warnings.catch_warnings(record=True) as caught_warnings:
403+
result = gl.http_list("/projects", all=True)
404+
assert len(caught_warnings) == 0
405+
406+
assert isinstance(result, list)
407+
assert len(result) == 20
408+
assert responses.assert_call_count(url, 3) is True
409+
410+
346411
@responses.activate
347412
def test_list_request_404(gl 3CC0 ):
348413
url = "http://localhost/api/v4/not_there"

0 commit comments

Comments
 (0)
0