8000 feat: emit a warning when using a `list()` method returns max · python-gitlab/python-gitlab@9d1e6e8 · GitHub
[go: up one dir, main page]
Skip to content

Commit 9d1e6e8

Browse files
JohnVillalovosJohnVillalovos
committed
feat: emit a warning when using a list() method returns max
A common cause of issues filed and questions raised is that a user will call a `list()` method and only get 20 items. As this is the default maximum of items that will be returned from a `list()` method. To help with this we now emit a warning when the result from a `list()` method is greater-than or equal to 20 (or the specified `per_page` value) and the user is not using either `all=True`, `as_list=False`, or `page=X`.
1 parent 64d01ef commit 9d1e6e8

31 files changed

+145
-6
lines changed

‎gitlab/client.py

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@
1818

1919
import os
2020
import time
21+
import warnings
2122
from typing import Any, cast, Dict, List, Optional, Tuple, TYPE_CHECKING, Union
2223

2324
import requests
2425
import requests.utils
2526
from requests_toolbelt.multipart.encoder import MultipartEncoder # type: ignore
2627

28+
import gitlab
2729
import gitlab.config
2830
import gitlab.const
2931
import gitlab.exceptions
@@ -35,6 +37,12 @@
3537
"{source!r} to {target!r}"
3638
)
3739

40+
# https://docs.gitlab.com/ee/api/#offset-based-pagination
41+
_PAGINATION_URL = (
42+
f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
43+
f"api-usage.html#pagination"
44+
)
45+
3846

3947
class Gitlab:
4048
"""Represents a GitLab server connection.
@@ -818,7 +826,26 @@ def http_list(
818826

819827
if page or as_list is True:
820828
# pagination requested, we return a list
821-
return list(GitlabList(self, url, query_data, get_next=False, **kwargs))
829+
gl_list = GitlabList(self, url, query_data, get_next=False, **kwargs)
830+
items = list(gl_list)
831+
if (
832+
page is None
833+
and len(items) >= gl_list.per_page
834+
and (gl_list.total is None or len(items) < gl_list.total)
835+
):
836+
total_items = "10,000+" if gl_list.total is None else gl_list.total
837+
# Warn the user that they are only going to retrieve `per_page` maximum
838+
# items. This is a common cause of issues filed.
839+
warnings.warn(
840+
(
841+
f"Calling a `list()` method without specifying `all=True` or "
842+
f"`as_list=False` will return a maximum of {gl_list.per_page} "
843+
f"items. Your query returned {len(items)} of {total_items} "
844+
f"items. See {_PAGINATION_URL} for more details"
845+
),
846+
UserWarning,
847+
)
848+
return items
822849

823850
# No pagination, generator requested
824851
return GitlabList(self, url, query_data, **kwargs)

‎tests/functional/api/test_gitlab.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,13 @@ def test_template_dockerfile(gl):
8181

8282

8383
def test_template_gitignore(gl):
84-
assert gl.gitignores.list()
84+
assert gl.gitignores.list(all=True)
8585
gitignore = gl.gitignores.get("Node")
8686
assert gitignore.content is not None
8787

8888

8989
def test_template_gitlabciyml(gl):
90-
assert gl.gitlabciymls.list()
90+
assert gl.gitlabciymls.list(all=True)
9191
gitlabciyml = gl.gitlabciymls.get("Nodejs")
9292
assert gitlabciyml.content is not None
9393

‎tests/unit/objects/test_audit_events.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ def resp_list_audit_events():
5252
url=audit_events_url,
5353
json=[audit_events_content],
5454
content_type="application/json",
55+
headers={"X-Total": "1", "x-per-page": "20"},
5556
status=200,
5657
)
5758
yield rsps

‎tests/unit/objects/test_badges.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ def resp_list_badges():
6363
json=[badge_content],
6464
content_type="application/json",
6565
status=200,
66+
headers={"X-Total": "1", "x-per-page": "20"},
6667
)
6768
yield rsps
6869

‎tests/unit/objects/test_bridges.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,15 @@ def resp_list_bridges():
8484
url="http://localhost/api/v4/projects/1/pipelines/6/bridges",
8585
json=[export_bridges_content],
8686
content_type="application/json",
87+
headers={"X-Total": "1", "x-per-page": "20"},
8788
status=200,
8889
)
8990
rsps.add(
9091
method=responses.GET,
9192
url="http://localhost/api/v4/projects/1/pipelines",
9293
json=export_pipelines_content,
9394
content_type="application/json",
95+
headers={"X-Total": "1", "x-per-page": "20"},
9496
status=200,
9597
)
9698
yield rsps

‎tests/unit/objects/test_group_access_tokens.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ def resp_list_group_access_token():
2727
url="http://localhost/api/v4/groups/1/access_tokens",
2828
json=content,
2929
content_type="application/json",
30+
headers={"X-Total": "1", "x-per-page": "20"},
3031
status=200,
3132
)
3233
yield rsps
@@ -84,6 +85,7 @@ def resp_revoke_group_access_token():
8485
url="http://localhost/api/v4/groups/1/access_tokens",
8586
json=content,
8687
content_type="application/json",
88+
headers={"X-Total": "1", "x-per-page": "20"},
8789
status=200,
8890
)
8991
yield rsps

‎tests/unit/objects/test_groups.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ def resp_list_subgroups_descendant_groups():
7777
),
7878
json=subgroup_descgroup_content,
7979
content_type="application/json",
80+
headers={"X-Total": "1", "x-per-page": "20"},
8081
status=200,
8182
)
8283
yield rsps

‎tests/unit/objects/test_hooks.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ def resp_hooks_list():
3737
url=re.compile(r"http://localhost/api/v4/((groups|projects)/1/|)hooks"),
3838
json=hooks_content,
3939
content_type="application/json",
40+
headers={"X-Total": "1", "x-per-page": "20"},
4041
status=200,
4142
)
4243
yield rsps

‎tests/unit/objects/test_issues.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ def resp_list_issues():
2323
url="http://localhost/api/v4/issues",
2424
json=content,
2525
content_type="application/json",
26+
headers={"X-Total": "2", "x-per-page": "20"},
2627
status=200,
2728
)
2829
yield rsps

‎tests/unit/objects/test_members.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ def resp_list_billable_group_members():
2929
url="http://localhost/api/v4/groups/1/billable_members",
3030
json=billable_members_content,
3131
content_type="application/json",
32+
headers={"X-Total": "1", "x-per-page": "20"},
3233
status=200,
3334
)
3435
yield rsps

‎tests/unit/objects/test_merge_request_pipelines.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ def resp_list_merge_request_pipelines():
2222
url="http://localhost/api/v4/projects/1/merge_requests/1/pipelines",
2323
json=[pipeline_content],
2424
content_type="application/json",
25+
headers={"X-Total": "1", "x-per-page": "20"},
2526
status=200,
2627
)
2728
yield rsps

‎tests/unit/objects/test_merge_requests.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ def resp_list_merge_requests():
3838
),
3939
json=[mr_content],
4040
content_type="application/json",
41+
headers={"X-Total": "1", "x-per-page": "20"},
4142
status=200,
4243
)
4344
yield rsps

‎tests/unit/objects/test_merge_trains.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ def resp_list_merge_trains():
5454
url="http://localhost/api/v4/projects/1/merge_trains",
5555
json=[mr_content],
5656
content_type="application/json",
57+
headers={"X-Total": "1", "x-per-page": "20"},
5758
status=200,
5859
)
5960
yield rsps

‎tests/unit/objects/test_packages.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ def resp_list_packages():
118118
url=re.compile(r"http://localhost/api/v4/(groups|projects)/1/packages"),
119119
json=[package_content],
120120
content_type="application/json",
121+
headers={"X-Total": "1", "x-per-page": "20"},
121122
status=200,
122123
)
123124
yield rsps
@@ -172,6 +173,7 @@ def resp_delete_package_file_list(no_content):
172173
),
173174
json=package_file_content,
174175
content_type="application/json",
176+
headers={"X-Total": "3", "x-per-page": "20"},
175177
status=200,
176178
)
177179
for pkg_file_id in range(25, 28):
@@ -195,6 +197,7 @@ def resp_list_package_files():
195197
),
196198
json=package_file_content,
197199
content_type="application/json",
200+
headers={"X-Total": "3", "x-per-page": "20"},
198201
status=200,
199202
)
200203
yield rsps

‎tests/unit/objects/test_personal_access_tokens.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ def resp_personal_access_token(no_content):
4848
url=token_url,
4949
json=[content],
5050
content_type="application/json",
51+
headers={"X-Total": "1", "x-per-page": "20"},
5152
status=200,
5253
)
5354
rsps.add(

‎tests/unit/objects/test_project_access_tokens.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ def resp_list_project_access_token():
2727
url="http://localhost/api/v4/projects/1/access_tokens",
2828
json=content,
2929
content_type="application/json",
30+
headers={"X-Total": "1", "x-per-page": "20"},
3031
status=200,
3132
)
3233
yield rsps
@@ -84,6 +85,7 @@ def resp_revoke_project_access_token():
8485
url="http://localhost/api/v4/projects/1/access_tokens",
8586
json=content,
8687
content_type="application/json",
88+
headers={"X-Total": "1", "x-per-page": "20"},
8789
status=200,
8890
)
8991
yield rsps

‎tests/unit/objects/test_project_merge_request_approvals.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ def resp_snippet():
193193
url="http://localhost/api/v4/projects/1/merge_requests",
194194
json=merge_request_content,
195195
content_type="application/json",
196+
headers={"X-Total": "1", "x-per-page": "20"},
196197
status=200,
197198
)
198199
rsps.add(
@@ -207,6 +208,7 @@ def resp_snippet():
207208
url="http://localhost/api/v4/projects/1/merge_requests/1/approval_rules",
208209
json=mr_ars_content,
209210
content_type="application/json",
211+
headers={"X-Total": "1", "x-per-page": "20"},
210212
status=200,
211213
)
212214
rsps.add(

‎tests/unit/objects/test_projects.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ def resp_list_projects():
3636
url="http://localhost/api/v4/projects",
3737
json=[project_content],
3838
content_type="application/json",
39+
headers={"X-Total": "1", "x-per-page": "20"},
3940
status=200,
4041
)
4142
yield rsps

‎tests/unit/objects/test_releases.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ def resp_list_links():
5353
url=links_url,
5454
json=[link_content],
5555
content_type="application/json",
56+
headers={"X-Total": "1", "x-per-page": "20"},
5657
status=200,
5758
)
5859
yield rsps

‎tests/unit/objects/test_remote_mirrors.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ def resp_remote_mirrors():
2828
url="http://localhost/api/v4/projects/1/remote_mirrors",
2929
json=[content],
3030
content_type="application/json",
31+
headers={"X-Total": "1", "x-per-page": "20"},
3132
status=200,
3233
)
3334
rsps.add(

‎tests/unit/objects/test_resource_label_events.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,15 @@ def resp_group_epic_request_label_events():
2222
url="http://localhost/api/v4/groups/1/epics",
2323
json=[epic_content],
2424
content_type="application/json",
25+
headers={"X-Total": "1", "x-per-page": "20"},
2526
status=200,
2627
)
2728
rsps.add(
2829
method=responses.GET,
2930
url="http://localhost/api/v4/groups/1/epics/1/resource_label_events",
3031
json=[events_content],
3132
content_type="application/json",
33+
headers={"X-Total": "1", "x-per-page": "20"},
3234
status=200,
3335
)
3436
yield rsps
@@ -44,13 +46,15 @@ def resp_merge_request_label_events():
4446
url="http://localhost/api/v4/projects/1/merge_requests",
4547
json=[mr_content],
4648
content_type="application/json",
49+
headers={"X-Total": "1", "x-per-page": "20"},
4750
status=200,
4851
)
4952
rsps.add(
5053
method=responses.GET,
5154
url="http://localhost/api/v4/projects/1/merge_requests/1/resource_label_events",
5255
json=[events_content],
5356
content_type="application/json",
57+
headers={"X-Total": "1", "x-per-page": "20"},
5458
status=200,
5559
)
5660
yield rsps
@@ -66,13 +70,15 @@ def resp_project_issue_label_events():
6670
url="http://localhost/api/v4/projects/1/issues",
6771
json=[issue_content],
6872
content_type="application/json",
73+
headers={"X-Total": "1", "x-per-page": "20"},
6974
status=200,
7075
)
7176
rsps.add(
7277
method=responses.GET,
7378
url="http://localhost/api/v4/projects/1/issues/1/resource_label_events",
7479
json=[events_content],
7580
content_type="application/json",
81+
headers={"X-Total": "1", "x-per-page": "20"},
7682
status=200,
7783
)
7884
yield rsps

‎tests/unit/objects/test_resource_milestone_events.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ def resp_merge_request_milestone_events():
2121
url="http://localhost/api/v4/projects/1/merge_requests",
2222
json=[mr_content],
2323
content_type="application/json",
24+
headers={"X-Total": "1", "x-per-page": "20"},
2425
status=200,
2526
)
2627
rsps.add(
2728
method=responses.GET,
2829
url="http://localhost/api/v4/projects/1/merge_requests/1/resource_milestone_events",
2930
json=[events_content],
3031
content_type="application/json",
32+
headers={"X-Total": "1", "x-per-page": "20"},
3133
status=200,
3234
)
3335
yield rsps
@@ -43,13 +45,15 @@ def resp_project_issue_milestone_events():
4345
url="http://localhost/api/v4/projects/1/issues",
4446
json=[issue_content],
4547
content_type="application/json",
48+
headers={"X-Total": "1", "x-per-page": "20"},
4649
status=200,
4750
)
4851
rsps.add(
4952
method=responses.GET,
5053
url="http://localhost/api/v4/projects/1/issues/1/resource_milestone_events",
5154
json=[events_content],
5255
content_type="application/json",
56+
headers={"X-Total": "1", "x-per-page": "20"},
5357
status=200,
5458
)
5559
yield rsps

‎tests/unit/objects/test_resource_state_events.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ def resp_list_project_issue_state_events():
2222
url="http://localhost/api/v4/projects/1/issues/1/resource_state_events",
2323
json=[issue_event_content],
2424
content_type="application/json",
25+
headers={"X-Total": "1", "x-per-page": "20"},
2526
status=200,
2627
)
2728
yield rsps
@@ -48,6 +49,7 @@ def resp_list_merge_request_state_events():
4849
url="http://localhost/api/v4/projects/1/merge_requests/1/resource_state_events",
4950
json=[mr_event_content],
5051
content_type="application/json",
52+
headers={"X-Total": "1", "x-per-page": "20"},
5153
status=200,
5254
)
5355
yield rsps

‎tests/unit/objects/test_runners.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ def resp_get_runners_jobs():
8787
url="http://localhost/api/v4/runners/6/jobs",
8888
json=runner_jobs,
8989
content_type="application/json",
90+
headers={"X-Total": "1", "x-per-page": "20"},
9091
status=200,
9192
)
9293
yield rsps
@@ -100,6 +101,7 @@ def resp_get_runners_list():
100101
url=re.compile(r".*?(/runners(/all)?|/(groups|projects)/1/runners)"),
101102
json=[runner_shortinfo],
102103
content_type="application/json",
104+
headers={"X-Total": "1", "x-per-page": "20"},
103105
status=200,
104106
)
105107
yield rsps

‎tests/unit/objects/test_services.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ def resp_service():
3838
url="http://localhost/api/v4/projects/1/services",
3939
json=[content],
4040
content_type="application/json",
41+
headers={"X-Total": "1", "x-per-page": "20"},
4142
status=200,
4243
)
4344
rsps.add(

‎tests/unit/objects/test_snippets.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ def resp_snippet():
2727
url="http://localhost/api/v4/projects/1/snippets",
2828
json=[content],
2929
content_type="application/json",
30+
headers={"X-Total": "1", "x-per-page": "20"},
3031
status=200,
3132
)
3233
rsps.add(

‎tests/unit/objects/test_todos.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ def resp_todo(json_content):
2525
url="http://localhost/api/v4/todos",
2626
json=json_content,
2727
content_type="application/json",
28+
headers={"X-Total": "1", "x-per-page": "20"},
2829
status=200,
2930
)
3031
rsps.add(

0 commit comments

Comments
 (0)
0