10000 Make --test and --include non-cumulative again. · robotframework/robotframework@6c6476d · GitHub
[go: up one dir, main page]
Skip to content

Commit 6c6476d

Browse files
pekkaklarckpekkaklarck
committed
Make --test and --include non-cumulative again.
These options were made cumulative in RF 7.0 (#4721), but that behavior caused backwards incompatibility problems with Pabot and with --rerunfailed. The benefits of the change weren't that big and the behavior was changed back to original in RF 7.0.1. Fixes #5023.
1 parent b7b8c6f commit 6c6476d

File tree

10 files changed

+58
-51
lines changed

10 files changed

+58
-51
lines changed

‎atest/robot/cli/runner/rerunfailed.robot

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ Suite initialization
4949
Run Tests ${EMPTY} ${SUITE DIR}
5050
Copy File ${OUTFILE} ${RUN FAILED FROM}
5151
Copy File ${ORIG DIR}/runfailed2.robot ${SUITE DIR}/runfailed.robot
52-
Run Tests --rerunfailed ${RUN FAILED FROM} --test Selected --exclude excluded_tag ${SUITE DIR}
52+
Run Tests --rerunfailed ${RUN FAILED FROM} --test Selected --include common --exclude excluded_tag ${SUITE DIR}
5353

5454
Test Should Have Been Executed
5555
[Arguments] ${name}

‎atest/robot/core/filter_by_names.robot

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,10 @@ ${SUITE DIR} misc/suites
1717
Run And Check Tests --test *one --test Fi?st First Second One Third One
1818
Run And Check Tests --test [Great]Lob[sterB]estCase[!3-9] GlobTestCase1 GlobTestCase2
1919

20-
--test is cumulative with --include
21-
Run And Check Tests --test fifth --include t1 First Fifth
22-
23-
--exclude wins ovet --test
24-
Run And Check Tests --test fi* --exclude t1 Fifth
25-
2620
--test not matching
2721
Run Failing Test
2822
... Suite 'Many Tests' contains no tests matching name 'notexists'.
29-
... --test notexists ${SUITE FILE}
23+
... --test notexists
3024

3125
--test not matching with multiple inputs
3226
Run Failing Test
@@ -36,6 +30,18 @@ ${SUITE DIR} misc/suites
3630
... Suite 'My Name' contains no tests matching name 'notexists'.
3731
... --name "My Name" --test notexists ${SUITE FILE} ${SUITE DIR}
3832

33+
--test and --include must both match
34+
Run And Check Tests --test first --include t1 -i f1 First
35+
Run Failing Test
36+
... Suite 'Many Tests' contains no tests matching name 'fifth' and matching tag 't1'.
37+
... --test fifth --include t1
38+
39+
--exclude wins over --test
40+
Run And Check Tests --test fi* --exclude t1 Fifth
41+
Run Failing Test
42+
... Suite 'Many Tests' contains no tests matching name 'first' and not matching tag 'f1'.
43+
... --test first --exclude f1
44+
3945
--suite once
4046
Run Suites --suite tsuite1
4147
Should Contain Suites ${SUITE} TSuite1
@@ -131,7 +137,7 @@ Parent suite init files are processed
131137
Should Contain Tests ${SUITE} Suite1 First Suite3 First
132138

133139
--suite, --test, --include and --exclude
134-
Run Suites --suite sub* --suite "custom name *" --test *first -s nomatch -t nomatch --include sub3 --exclude t1
140+
Run Suites --suite sub* --suite "custom name *" --test "subsuite3 second" -t *first -s nomatch -t nomatch --include f1 --exclude t1
135141
Should Contain Suites ${SUITE} Custom name for 📂 'subsuites2' Subsuites
136142
Should Contain Tests ${SUITE} SubSuite2 First SubSuite3 Second
137143

@@ -162,6 +168,6 @@ Run Suites
162168
Stderr Should Be Empty
163169

164170
Run Failing Test
165-
[Arguments] ${error} ${options} ${sources}
171+
[Arguments] ${error} ${options} ${sources}=${SUITE FILE}
166172
Run Tests Without Processing Output ${options} ${sources}
167173
Stderr Should Be Equal To [ ERROR ] ${error}${USAGE TIP}\n

‎atest/robot/rebot/filter_by_names.robot

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,10 @@ ${INPUT FILE} %{TEMPDIR}${/}robot-test-file.xml
2222
Run And Check Tests --test *one --test Fi?st First Second One Third One
2323
Run And Check Tests --test [Great]Lob[sterB]estCase[!3-9] GlobTestCase1 GlobTestCase2
2424

25-
--test is cumulative with --include
26-
Run And Check Tests --test fifth --include t2 First Fifth Suite1 Second SubSuite3 Second
27-
28-
--exclude wins ovet --test
29-
Run And Check Tests --test fi* --exclude t1 Fifth
30-
3125
--test not matching
3226
Failing Rebot
3327
... Suite 'Root' contains no tests matching name 'nonex'.
34-
... --test nonex ${INPUT FILE}
28+
... --test nonex
3529

3630
--test not matching with multiple inputs
3731
Failing Rebot
@@ -41,6 +35,18 @@ ${INPUT FILE} %{TEMPDIR}${/}robot-test-file.xml
4135
... Suite 'My Name' contains no tests matching name 'nonex'.
4236
... --test nonex -N "My Name" ${INPUT FILE} ${INPUT FILE}
4337

38+
--test and --include must both match
39+
Run And Check Tests --test first --include t1 -i f1 First
40+
Failing Rebot
41+
... Suite 'Root' contains no tests matching name 'fifth' and matching tag 't1'.
42+
... --test fifth --include t1
43+
44+
--exclude wins over --test
45+
Run And Check Tests --test fi* --exclude t1 Fifth
46+
Failing Rebot
47+
... Suite 'Root' contains no tests matching name 'first' and not matching tag 'f1'.
48+
... --test first --exclude f1
49+
4450
--suite once
4551
Run And Check Suites --suite tsuite1 Tsuite1
4652

@@ -96,7 +102,7 @@ ${INPUT FILE} %{TEMPDIR}${/}robot-test-file.xml
96102
Should Contain Tests ${SUITE} Suite1 First Suite3 First
97103

98104
--suite, --test, --include and --exclude
99-
Run Suites --suite sub* --suite "custom name *" --test *first -s nomatch -t nomatch --include sub3 --exclude t1
105+
Run Suites --suite sub* --suite "custom name *" --test "subsuite3 second" -t *first -s nomatch -t nomatch --include f1 --exclude t1
100106
Should Contain Suites ${SUITE} Suites
101107
Should Contain Suites ${SUITE.suites[0]} Custom name for 📂 'subsuites2' Subsuites
102108
Should Contain Tests ${SUITE} SubSuite2 First SubSuite3 Second
@@ -158,6 +164,6 @@ Run Suites
158164
Stderr Should Be Empty
159165

160166
Failing Rebot
161-
[Arguments] ${error} ${options} ${sources}
167+
[Arguments] ${error} ${options} ${sources}=${INPUT FILE}
162168
Run Rebot Without Processing Output ${options} ${sources}
163169
Stderr Should Be Equal To [ ERROR ] ${error}${USAGE TIP}\n

‎atest/testdata/cli/runfailed/onlypassing/passing.robot

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
*** Test Cases ***
22
Passing
3-
No Operation
3+
No Operation

‎atest/testdata/cli/runfailed/suite/__init__.robot

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*** Settings ***
2+
Test Tags common

‎atest/testdata/cli/runfailed/suite/subsuite/suite_failed_with_excluded_tag.robot

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,5 @@ Suite Teardown No Operation
33

44
*** Test Cases ***
55
Failing with tag
6-
[Tags] excluded_tag
6+
[Tags] excluded_tag
77
Fail failed test
8-

‎doc/userguide/src/ExecutingTestCases/ConfiguringExecution.rst

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -295,25 +295,19 @@ many interesting possibilities:
295295

296296
Options :option:`--include` and :option:`--exclude` can be used in combination
297297
with :option:`--suite` and :option:`--test` discussed in the previous section.
298-
The general rules how they work together are as follows:
299-
300-
- If :option:`--suite` is used, tests must be in the specified suite in addition
301-
to satisfying other selection criteria.
302-
303-
- If :option:`--include` is used with :option:`--test`, it is enough for a test
304-
to match either of them.
305-
306-
- If :option:`--exclude` is used, tests matching it are never selected.
307-
308-
The above rules are demonstrated in the following examples::
298+
In that case tests that are selected must match all selection criteria::
309299

310300
--suite example --include tag # Match test if it is in suite 'example' and has tag 'tag'.
311301
--suite example --exclude tag # Match test if it is in suite 'example' and does not have tag 'tag'.
312-
--test example --include tag # Match test if it has name 'example' or it has tag 'tag'.
302+
--test ex* --include tag # Match test if its name starts with 'ex' and it has tag 'tag'.
313303
--test ex* --exclude tag # Match test if its name starts with 'ex' and it does not have tag 'tag'.
314304

315-
.. note:: Prior to Robot Framework 7.0 using `--include` and `--test` together
316-
required test to have both a matching tag and a matching name.
305+
.. note:: In Robot Framework 7.0 `--include` and `--test` were cumulative and
306+
selected tests needed to match only either of these options. That behavior
307+
caused `backwards incompatibility problems`__ and it was changed
308+
back to the original already in Robot Framework 7.0.1.
309+
310+
__ https://github.com/robotframework/robotframework/issues/5023
317311

318312
Re-executing failed test cases
319313
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -366,7 +360,7 @@ When no tests match selection
366360
By default when no tests match the selection criteria test execution fails
367361
with an error like::
368362

369-
[ ERROR ] Suite 'Example' with includes 'xxx' contains no test cases.
363+
[ ERROR ] Suite 'Example' contains no tests matching tag 'xxx'.
370364

371365
Because no outputs are generated, this behavior can be problematic if tests
372366
are executed and results processed automatically. Luckily a command line

‎src/robot/model/configurer.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ def _get_test_selector_msgs(self):
7373
parts = []
7474
for separator, explanation, selectors in [
7575
(None, 'matching name', self.include_tests),
76-
('or', 'matching tags', self.include_tags),
76+
('and', 'matching tags', self.include_tags),
7777
('and', 'not matching tags', self.exclude_tags)
7878
]:
7979
if selectors:

‎src/robot/model/filter.py

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ def start_suite(self, suite: 'TestSuite'):
8484
suite.start_time = suite.end_time = suite.elapsed_time = None
8585
if self.include_suites is not None:
8686
return self._filter_based_on_suite_name(suite)
87-
suite.tests = [t for t in suite.tests if self._test_included(t)]
87+
self._filter_tests(suite)
8888
return bool(suite.suites)
8989

9090
def _filter_based_on_suite_name(self, suite: 'TestSuite') -> bool:
@@ -96,16 +96,16 @@ def _filter_based_on_suite_name(self, suite: 'TestSuite') -> bool:
9696
suite.tests = []
9797
return True
9898

99-
def _test_included(self, test: 'TestCase') -> bool:
99+
def _filter_tests(self, suite: 'TestSuite'):
100100
tests, include, exclude \
101101
= self.include_tests, self.include_tags, self.exclude_tags
102-
if exclude is not None and exclude.match(test.tags):
103-
return False
104-
if include is not None and include.match(test.tags):
105-
return True
106-
if tests is not None and tests.match(test.name, test.full_name):
107-
return True
108-
return include is None and tests is None
102+
t: TestCase
103+
if tests is not None:
104+
suite.tests = [t for t in suite.tests if tests.match(t.name, t.full_name)]
105+
if include is not None:
106+
suite.tests = [t for t in suite.tests if include.match(t.tags)]
107+
if exclude is not None:
108+
suite.tests = [t for t in suite.tests if not exclude.match(t.tags)]
109109

110110
def __bool__(self) -> bool:
111111
return bool(self.include_suites is not None or

‎utest/result/test_configurer.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
from itertools import chain
21
import unittest
3-
from robot.utils.asserts import assert_equal, assert_raises_with_msg, assert_true
2+
from itertools import chain
43

54
from robot.errors import DataError
65
from robot.result import Keyword, TestCase, TestSuite
76
from robot.result.configurer import SuiteConfigurer
7+
from robot.utils.asserts import assert_equal, assert_raises_with_msg, assert_true
88

99

1010
SETUP = Keyword.SETUP
@@ -90,7 +90,7 @@ def test_no_matching_tests_with_one_selector_each(self):
9090
assert_raises_with_msg(
9191
DataError,
9292
"Suite 'root' contains no tests matching name 't' "
93-
"or matching tag 'i' "
93+
"and matching tag 'i' "
9494
"and not matching tag 'e' "
9595
"in suite 's'.",
9696
self.suite.visit, configurer
@@ -104,7 +104,7 @@ def test_no_matching_tests_with_multiple_selectors(self):
104104
assert_raises_with_msg(
105105
DataError,
106106
"Suite 'root' contains no tests matching name 't1' or 't2' "
107-
"or matching tags 'i1', 'i2' or 'i3' "
107+
"and matching tags 'i1', 'i2' or 'i3' "
108108
"and not matching tags 'e1' or 'e2' "
109109
"in suites 's1', 's2' or 's3'.",
110110
self.suite.visit, configurer

0 commit comments

Comments
 (0)
0