8000 tests: try and get pipeline green again (#2259) · nginx/kubernetes-ingress@c4a38b1 · GitHub
[go: up one dir, main page]
Skip to content

Commit c4a38b1

Browse files
ciarams87ciarams87
authored
tests: try and get pipeline green again (#2259)
* Test changes * Try using flaky * Mark vsr external route events as xfail; use single syslog pod for AP sec log tests.
1 parent 69ad025 commit c4a38b1

16 files changed

+125
-50
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,12 +159,12 @@ jobs:
159159
{\"image\": \"opentracing\", \"marker\": \"vs\"}, \
160160
{\"image\": \"ubi\", \"marker\": \"ts\"}, \
161161
{\"image\": \"debian-plus\", \"marker\": \"vs\"}, \
162+
{\"image\": \"debian-plus\", \"marker\": \"ts\"}, \
162163
{\"image\": \"alpine-plus\", \"marker\":\"ingresses\"}, \
163164
{\"image\": \"opentracing-plus\", \"marker\": \"vsr\"}, \
164165
{\"image\": \"ubi-plus\", \"marker\": \"policies\"}]}"
165166
else
166-
echo "::set-output name=matrix::{\"k8s\": [\"1.19.11\", \"1.20.7\", \"1.21.2\", \"1.22.4\", \"1.23.0\"], \
167-
\"images\": [{\"image\": \"debian\"}, {\"image\": \"debian-plus\"}]}"
167+
echo "::set-output name=matrix::{\"k8s\": [\"1.19.11\", \"1.20.7\", \"1.21.2\", \"1.22.4\", \"1.23.0\"]}"
168168
fi
169169
170170
smoke-tests:

‎tests/conftest.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,11 @@ def pytest_collection_modifyitems(config, items) -> None:
112112
for item in items:
113113
if "skip_for_nginx_plus" in item.keywords:
114114
item.add_marker(skip_for_nginx_plus)
115+
if config.getoption("--service") == "loadbalancer":
116+
skip_for_loadbalancer = pytest.mark.skip(reason="Skip a test for loadbalancer service")
117+
for item in items:
118+
if "skip_for_loadbalancer" in item.keywords:
119+
item.add_marker(skip_for_loadbalancer)
115120
if "-ap" not in config.getoption("--image"):
116121
appprotect = pytest.mark.skip(reason="Skip AppProtect test in non-AP image")
117122
for item in items:

‎tests/data/rate-limit/policies/rate-limit-secondary.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ metadata:
44
name: rate-limit-secondary
55
spec:
66
rateLimit:
7-
rate: 10r/s
7+
rate: 5r/s
88
key: ${binary_remote_addr}
99
zoneSize: 10M

‎tests/docker/gitlab.Dockerfile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,18 @@
22
FROM python:3.9
33

44
ARG GCLOUD_VERSION=364.0.0
5+
ARG HELM_VERSION=3.5.4
56

67
RUN apt-get update && apt-get install -y curl git jq \
78
&& curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
89
&& chmod +x ./kubectl \
910
&& mv ./kubectl /usr/local/bin \
1011
&& curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz \
1112
&& tar xvzf google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz \
12-
&& mv google-cloud-sdk /usr/lib/
13+
&& mv google-cloud-sdk /usr/lib/ \
14+
&& curl -LO https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz \
15+
&& tar -zxvf helm-v${HELM_VERSION}-linux-amd64.tar.gz \
16+
&& mv linux-amd64/helm /usr/local/bin/helm
1317

1418
WORKDIR /workspace/tests
1519

‎tests/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ more-itertools==8.12.0
1414
mock==4.0.3
1515
grpcio==1.42.0
1616
grpcio-tools==1.42.0
17+
flaky==3.7.0

‎tests/suite/resources_utils.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,19 @@ def get_pods_amount(v1: CoreV1Api, namespace) -> int:
320320
pods = v1.list_namespaced_pod(namespace)
321321
return 0 if not pods.items else len(pods.items)
322322

323+
def get_pod_name_that_contains(v1: CoreV1Api, namespace, contains_string) -> str:
324+
"""
325+
Get an amount of pods.
326+
327+
:param v1: CoreV1Api
328+
:param namespace: namespace
329+
:param contains_string: string to search on
330+
:return: string
331+
"""
332+
for item in v1.list_namespaced_pod(namespace).items:
333+
if contains_string in item.metadata.name:
334+
return item.metadata.name
335+
return ""
323336

324337
def create_service_from_yaml(v1: CoreV1Api, namespace, yaml_manifest) -> str:
325338
"""
@@ -808,6 +821,28 @@ def get_file_contents(v1: CoreV1Api, file_path, pod_name, pod_namespace) -> str:
808821
return result_conf
809822

810823

824+
def clear_file_contents(v1: CoreV1Api, file_path, pod_name, pod_namespace):
825+
"""
826+
Execute 'cat /dev/null > file_path' command in a pod.
827+
828+
:param v1: CoreV1Api
829+
:param pod_name: pod name
830+
:param pod_namespace: pod namespace
831+
:param file_path: an absolute path to a file in the pod
832+
"""
833+
command = ["cat /dev/null > ", file_path]
834+
resp = stream(
835+
v1.connect_get_namespaced_pod_exec,
836+
pod_name,
837+
pod_namespace,
838+
command=command,
839+
stderr=True,
840+
stdin=False,
841+
stdout=True,
842+
tty=False,
843+
)
844+
845+
811846
def get_ingress_nginx_template_conf(
812847
v1: CoreV1Api, ingress_namespace, ingress_name, pod_name, pod_namespace
813848
) -> str:

‎tests/suite/test_app_protect_grpc.py‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ def fin():
123123

124124
@pytest.mark.skip_for_nginx_oss
125125
@pytest.mark.appprotect
126+
@pytest.mark.flaky(max_runs=3)
126127
@pytest.mark.parametrize(
127128
"crd_ingress_controller_with_ap",
128129
[{"extra_args": [f"-enable-custom-resources", f"-enable-app-protect"]}],
@@ -161,8 +162,16 @@ def test_responses_grpc_block(
161162
# grpc.RpcError is also grpc.Call https://grpc.github.io/grpc/python/grpc.html#client-side-context
162163
ex = e.details()
163164
print(ex)
164-
165-
log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
165+
166+
log_contents = ""
167+
retry = 0
168+
while "ASM:attack_type" not in log_contents and retry <= 30:
169+
log_contents = get_file_contents(
170+
kube_apis.v1, log_loc, syslog_pod, test_namespace)
171+
retry += 1
172+
wait_before_test(1)
173+
print(f"Security log not updated, retrying... #{retry}")
174+
166175
assert (
167176
invalid_resp_text in ex and
168177
'ASM:attack_type="Directory Indexing"' in log_contents and
@@ -195,8 +204,16 @@ def test_responses_grpc_allow(
195204
except grpc.RpcError as e:
196205
print(e.details())
197206
pytest.fail("RPC error was not expected during call, exiting...")
198-
199-
log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
207+
208+
log_contents = ""
209+
retry = 0
210+
while "ASM:attack_type" not in log_contents and retry <= 30:
211+
log_contents = get_file_contents(
212+
kube_apis.v1, log_loc, syslog_pod, test_namespace)
213+
retry += 1
214+
wait_before_test(1)
215+
print(f"Security log not updated, retrying... #{retry}")
216+
200217
assert (
201218
valid_resp_txt in response.message and
202219
'ASM:attack_type="N/A"' in log_contents and

‎tests/suite/test_app_protect_integration.py‎

Lines changed: 19 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@
1717
get_file_contents, get_first_pod_name,
1818
get_ingress_nginx_template_conf,
1919
get_last_reload_time, get_pods_amount,
20-
get_service_endpoint, get_test_file_name,
20+
clear_file_contents, get_test_file_name,
2121
scale_deployment, wait_before_test,
2222
wait_until_all_pods_are_ready,
23-
write_to_json)
23+
write_to_json, get_pod_name_that_contains)
2424
from suite.yaml_utils import get_first_ingress_host_from_yaml
2525

2626
src_ing_yaml = f"{TEST_DATA}/appprotect/appprotect-ingress.yaml"
@@ -84,8 +84,13 @@ def appprotect_setup(
8484
src_pol_yaml = f"{TEST_DATA}/appprotect/{ap_policy}.yaml"
8585
pol_name = create_ap_policy_from_yaml(kube_apis.custom_objects, src_pol_yaml, test_namespace)
8686

87+
print("------------------------- Deploy syslog server ---------------------------")
88+
src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
89+
create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
90+
8791
def fin():
8892
print("Clean up:")
93+
delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
8994
delete_ap_policy(kube_apis.custom_objects, pol_name, test_namespace)
9095
delete_ap_logconf(kube_apis.custom_objects, log_name, test_namespace)
9196
delete_common_app(kube_apis, "simple", test_namespace)
@@ -297,6 +302,7 @@ def test_ap_enable_false_policy_incorrect(
297302
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
298303
assert_valid_responses(response)
299304

305+
@pytest.mark.flaky(max_runs=3)
300306
def test_ap_sec_logs_on(
301307
self,
302308
request,
@@ -309,15 +315,9 @@ def test_ap_sec_logs_on(
309315
"""
310316
Test corresponding log entries with correct policy (includes setting up a syslog server as defined in syslog.yaml)
311317
"""
312-
src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
313318
log_loc = "/var/log/messages"
314-
315-
create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
316-
317319
syslog_dst = f"syslog-svc.{test_namespace}"
318-
319-
# items[-1] because syslog pod is last one to spin-up
320-
syslog_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-1].metadata.name
320+
syslog_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace, "syslog-")
321321

322322
create_ingress_with_ap_annotations(
323323
kube_apis, src_ing_yaml, test_namespace, ap_policy, "True", "True", f"{syslog_dst}:514"
@@ -333,9 +333,9 @@ def test_ap_sec_logs_on(
333333
appprotect_setup.req_url + "/<script>", headers={"host": ingress_host}, verify=False
334334
)
335335
print(response_block.text)
336-
log_contents = ""
336+
log_contents_block = ""
337337
retry = 0
338-
while "ASM:attack_type" not in log_contents and retry <= 60:
338+
while "ASM:attack_type" not in log_contents_block and retry <= 30:
339339
log_contents_block = get_file_contents(
340340
kube_apis.v1, log_loc, syslog_pod, test_namespace
341341
)
@@ -354,7 +354,7 @@ def test_ap_sec_logs_on(
354354
log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
355355

356356
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
357-
delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
357+
clear_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
358358

359359
assert_invalid_responses(response_block)
360360
assert (
@@ -384,15 +384,8 @@ def test_ap_pod_startup(
384384
"""
385385
Log pod startup time while scaling up from 0 to 1
386386
"""
387-
src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
388-
create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
389-
390387
syslog_dst = f"syslog-svc.{test_namespace}"
391388

392-
# FIXME this is not used
393-
# items[-1] because syslog pod is last one to spin-up
394-
# syslog_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-1].metadata.name
395-
396389
create_ingress_with_ap_annotations(
397390
kube_apis, src_ing_yaml, test_namespace, ap_policy, "True", "True", f"{syslog_dst}:514"
398391
)
@@ -408,29 +401,27 @@ def test_ap_pod_startup(
408401
wait_before_test()
409402
num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 1)
410403
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
411-
delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
412404

413405
assert num is None
414406

407+
@pytest.mark.flaky(max_runs=3)
415408
def test_ap_multi_sec_logs(
416409
self, request, kube_apis, crd_ingress_controller_with_ap, appprotect_setup, test_namespace
417410
):
418411
"""
419412
Test corresponding log entries with multiple log destinations (in this case, two syslog servers)
420413
"""
421-
src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
422414
src_syslog2_yaml = f"{TEST_DATA}/appprotect/syslog2.yaml"
423415
log_loc = "/var/log/messages"
424416

425-
print("Create two syslog servers")
426-
create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
417+
print("Create a second syslog server")
427418
create_items_from_yaml(kube_apis, src_syslog2_yaml, test_namespace)
428419

429420
syslog_dst = f"syslog-svc.{test_namespace}"
430421
syslog2_dst = f"syslog2-svc.{test_namespace}"
431422

432-
syslog_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-2].metadata.name
433-
syslog2_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-1].metadata.name
423+
syslog_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace, "syslog-")
424+
syslog2_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace, "syslog2")
434425

435426
with open(src_ing_yaml) as f:
436427
doc = yaml.safe_load(f)
@@ -454,6 +445,7 @@ def test_ap_multi_sec_logs(
454445

455446
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
456447

448+
wait_before_test(30)
457449
ensure_response_from_backend(appprotect_setup.req_url, ingress_host, check404=True)
458450

459451
print("----------------------- Send request ----------------------")
@@ -480,8 +472,8 @@ def test_ap_multi_sec_logs(
480472
reload_times[f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
481473

482474
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
483-
delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
484475
delete_items_from_yaml(kube_apis, src_syslog2_yaml, test_namespace)
476+
clear_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
485477

486478
assert_invalid_responses(response)
487479
# check logs in dest. #1 i.e. syslog server #1
@@ -501,6 +493,7 @@ def test_ap_multi_sec_logs(
501493
and 'outcome="REJECTED"' in log2_contents
502494
)
503495

496+
@pytest.mark.flaky(max_runs=3)
504497
def test_ap_enable_true_policy_correct_uds(
505498
self, request, kube_apis, crd_ingress_controller_with_ap, appprotect_setup, test_namespace
506499
):

‎tests/suite/test_app_protect_waf_policies.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ def test_ap_waf_policy_allow(
310310
assert_valid_responses(response1)
311311
assert_valid_responses(response2)
312312

313+
@pytest.mark.flaky(max_runs=3)
313314
def test_ap_waf_policy_logs(
314315
self,
315316
kube_apis,
@@ -361,8 +362,15 @@ def test_ap_waf_policy_logs(
361362
headers={"host": virtual_server_setup.vs_host},
362363
)
363364
print(response.text)
364-
wait_before_test(5)
365-
log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
365+
log_contents = ""
366+
retry = 0
367+
while "ASM:attack_type" not in log_contents and retry <= 30:
368+
log_contents = get_file_contents(
369+
kube_apis.v1, log_loc, syslog_pod, test_namespace
370+
)
371+
retry += 1
372+
wait_before_test(1)
373+
print(f"Security log not updated, retrying... #{retry}")
366374

367375
delete_policy(kube_apis.custom_objects, "waf-policy", test_namespace)
368376
self.restore_default_vs(kube_apis, virtual_server_setup)

‎tests/suite/test_rl_policies.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,13 @@ def test_rl_policy_1rs(
103103
assert occur.count(200) <= 1
104104

105105
@pytest.mark.parametrize("src", [rl_vs_sec_src])
106-
def test_rl_policy_10rs(
106+
def test_rl_policy_5rs(
107107
self, kube_apis, crd_ingress_controller, virtual_server_setup, test_namespace, src,
108108
):
109109
"""
110-
Test if rate-limiting policy is working with 10 rps
110+
Test if rate-limiting policy is working with 5 rps
111111
"""
112-
rate_sec = 10
112+
rate_sec = 5
113113
print(f"Create rl policy")
114114
pol_name = create_policy_from_yaml(kube_apis.custom_objects, rl_pol_sec_src, test_namespace)
115115
print(f"Patch vs with policy: {src}")
@@ -251,10 +251,10 @@ def test_rl_override_spec_route(
251251
):
252252
"""
253253
List policies in vs spec and route resp. and test if route overrides spec
254-
route:policy = secondary (10 rps)
254+
route:policy = secondary (5 rps)
255255
spec:policy = primary (1 rps)
256256
"""
257-
rate_sec = 10
257+
rate_sec = 5
258258
print(f"Create rl policy")
259259
pol_name_pri = create_policy_from_yaml(
260260
kube_apis.custom_objects, rl_pol_pri_src, test_namespace

0 commit comments

Comments
 (0)
0