8000 Make --test and --cumulative. Fixes #4721. · robotframework/robotframework@57a74bc · GitHub
[go: up one dir, main page]
Skip to content

Commit 57a74bc

Browse files
pekkaklarckpekkaklarck
committed
Make --test and --cumulative. Fixes #4721.
Also enhance tests and documentation related to using --suite together with --test, --include and --exclude.
1 parent bd8b488 commit 57a74bc

File tree

9 files changed

+120
-64
lines changed

9 files changed

+120
-64
lines changed

‎atest/robot/core/filter_by_names.robot

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ ${SUITE DIR} misc/suites
1717
Run And Check Tests --test *one --test Fi?st First Second One Third One
1818
Run And Check Tests --test [Great]Lob[sterB]estCase[!3-9] GlobTestCase1 GlobTestCase2
1919

20+
--test is cumulative with --include
21+
Run And Check Tests --test fifth --include t1 First Fifth
22+
23+
--exclude wins ovet --test
24+
Run And Check Tests --test fi* --exclude t1 Fifth
25+
2026
--test not matching
2127
Run Failing Test
2228
... Suite 'Many Tests' contains no tests matching name 'notexists'.
@@ -109,25 +115,25 @@ Parent suite init files are processed
109115
... --suite xxx -N Custom ${SUITE DIR} ${SUITE FILE}
110116

111117
--suite and --test together
112-
[Documentation] Testing that only tests matching --test which are under suite matching --suite are run.
113-
Run Suites --suite subsuites --suite tsuite3 --test SubSuite1First
114-
Should Contain Suites ${SUITE} Subsuites
115-
Should Contain Tests ${SUITE} SubSuite1 First
118+
[Documentation] Validate that only tests matching --test under suites matching --suite are selected.
119+
Run Suites --suite suites.subsuites.sub2 --suite tsuite3 --test *First
120+
Should Contain Suites ${SUITE} Subsuites Tsuite3
121+
Should Contain Tests ${SUITE} SubSuite2 First Suite3 First
116122

117123
--suite and --test together not matching
118124
Run Failing Test
119125
... Suite 'Suites' contains no tests matching name 'Suite1*' or 'nomatch' in suites 'subsuites' or 'nomatch'.
120126
... --suite subsuites -s nomatch --test Suite1* -t nomatch ${SUITE DIR}
121127

122128
--suite with --include/--exclude
123-
Run Suites --suite tsuite? --include t? --exclude t2
124-
Should Contain Suites ${SUITE} Tsuite1 Tsuite2 Tsuite3
125-
Should Contain Tests ${SUITE} Suite1 First Suite2 First Suite3 First
126-
127-
--suite, --test, --inculde and --exclude
128-
Run Suites --suite sub* --test *first -s nosuite -t notest --include t1 --exclude sub3
129-
Should Contain Suites ${SUITE} Subsuites
130-
Should Contain Tests ${SUITE} SubSuite1 First
129+
Run Suites --suite tsuite[13] --include t? --exclude t2
130+
Should Contain Suites ${SUITE} Tsuite1 Tsuite3
131+
Should Contain Tests ${SUITE} Suite1 First Suite3 First
132+
133+
--suite, --test, --include and --exclude
134+
Run Suites --suite sub* --suite "custom name *" --test *first -s nomatch -t nomatch --include sub3 --exclude t1
135+
Should Contain Suites ${SUITE} Custom name for 📂 'subsuites2' Subsuites
136+
Should Contain Tests ${SUITE} SubSuite2 First SubSuite3 Second
131137

132138
--suite with long name and other filters
133139
Run Suites --suite suites.fourth --suite tsuite1 -s *.Subsuites.Sub1 --test *first* --exclude none

‎atest/robot/rebot/filter_by_names.robot

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ ${INPUT FILE} %{TEMPDIR}${/}robot-test-file.xml
2222
Run And Check Tests --test *one --test Fi?st First Second One Third One
2323
Run And Check Tests --test [Great]Lob[sterB]estCase[!3-9] GlobTestCase1 GlobTestCase2
2424

25+
--test is cumulative with --include
26+
Run And Check Tests --test fifth --include t2 First Fifth Suite1 Second SubSuite3 Second
27+
28+
--exclude wins ovet --test
29+
Run And Check Tests --test fi* --exclude t1 Fifth
30+
2531
--test not matching
2632
Failing Rebot
2733
... Suite 'Root' contains no tests matching name 'nonex'.
@@ -71,13 +77,30 @@ ${INPUT FILE} %{TEMPDIR}${/}robot-test-file.xml
7177
... --name CustomName --suite nonex ${INPUT FILE} ${INPUT FILE}
7278

7379
--suite and --test together
74-
Run And Check Suites and Tests --suite tsuite1 --suite tsuite3 --test *1first --test nomatch Tsuite1 Suite1 First
80+
[Documentation] Validate that only tests matching --test under suites matching --suite are selected.
81+
Run Suites --suite root.*.tsuite2 --suite manytests --test *first* --test nomatch --log log
82+
Should Contain Suites ${SUITE} Many Tests Suites
83+
Should Contain Tests ${SUITE.suites[0]} First
84+
Should Contain Tests ${SUITE.suites[1]} Suite2 First
85+
Check Stats
7586

7687
--suite and --test together not matching
7788
Failing Rebot
7889
... Suite 'Root' contains no tests matching name 'first', 'nonex' or '*one' in suites 'nonex' or 'suites'.
7990
... --suite nonex --suite suites --test first --test nonex --test *one ${INPUT FILE}
8091

92+
--suite with --include/--exclude
93+
Run Suites --suite tsuite[13] --include t? --exclude t2
94+
Should Contain Suites ${SUITE} Suites
95+
Should Contain Suites ${SUITE.suites[0]} Tsuite1 Tsuite3
96+
Should Contain Tests ${SUITE} Suite1 First Suite3 First
97+
98+
--suite, --test, --include and --exclude
99+
Run Suites --suite sub* --suite "custom name *" --test *first -s nomatch -t nomatch --include sub3 --exclude t1
100+
Should Contain Suites ${SUITE} Suites
101+
Should Contain Suites ${SUITE.suites[0]} Custom name for 📂 'subsuites2' Subsuites
102+
Should Contain Tests ${SUITE} SubSuite2 First SubSuite3 Second
103+
81104
Elapsed Time
82105
[Documentation] Test setting start, end and elapsed times correctly when filtering by tags
83106
# 1) Rebot hand-edited output with predefined times and check that times are read correctly. (A sanity check)
@@ -129,14 +152,6 @@ Run and Check Suites
129152
Should Contain Suites ${SUITE.suites[0]} @{suites}
130153
Check Stats
131154

132-
Run And Check Suites and Tests
133-
[Arguments] ${params} ${subsuite} @{tests}
134-
Run Suites ${params}
135-
Should Contain Suites ${SUITE.suites[0]} ${subsuite}
136-
Should Contain Tests ${SUITE} @{tests}
137-
Should Be True ${SUITE.statistics.passed} == len(@{tests})
138-
Check Stats
139-
140155
Run Suites
141156
[Arguments] ${options}
142157
Run Rebot ${options} ${INPUT FILE}

‎atest/robot/tags/include_and_exclude.robot

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
*** Settings ***
2+
Documentation Test --include and --exclude with Robot.
3+
...
4+
... These options working together with --suite and --test
5+
... is tested in filter_by_names.robot suite file.
26
Test Template Run And Check Include And Exclude
37
Resource atest_resource.robot
48

‎atest/robot/tags/include_and_exclude_with_rebot.robot

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
*** Settings ***
2-
Documentation Testing rebot's include/exclude functionality. Tests also include/exclude first during test execution and then with rebot.
2+
Documentation Test --include and --exclude with Rebot.
3+
...
4+
... These options working together with --suite and --test
5+
... is tested in filter_by_names.robot suite file.
36
Suite Setup Create Input Files
47
Suite Teardown Remove File ${INPUT FILE}
58
Test Template Run And Check Include And Exclude
@@ -9,7 +12,7 @@ Resource rebot_resource.robot
912
${TEST FILE} tags/include_and_exclude.robot
1013
${TEST FILE2} tags/no_force_no_default_tags.robot
1114
${INPUT FILE} %{TEMPDIR}/robot-tags-input.xml
12-
${INPUT FILE 2} %{TEMPDIR}/robot-tags-input-2.xml
15+
${INPUT FILE 2} %{TEMPDIR}/robot-tags-input-2.xml
1316
${INPUT FILES} ${INPUT FILE}
1417
@{INCL_ALL} Incl-1 Incl-12 Incl-123
1518
@{EXCL_ALL} excl-1 Excl-12 Excl-123

‎doc/userguide/src/ExecutingTestCases/ConfiguringExecution.rst

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -215,12 +215,6 @@ specified tests in specified suites are selected::
215215

216216
--suite mysuite --test mytest # Match test 'mytest' if its inside suite 'mysuite'.
217217

218-
Also this behavior `is likely to change`__ in the future and the above changed to mean
219-
selecting all tests in suite `mysuite` in addition to all tests with name `mytest`.
220-
A more reliable way to select a test in a suite is using `--test *.mysuite.mytest`
221-
or `--test *.mysuite.*.mytest` depending on should the test be directly inside
222-
the suite or not.
223-
224218
Using the :option:`--suite` option is more or less the same as executing
225219
the appropriate suite file or directory directly. The main difference is
226220
that if a file or directory is run directly, possible suite setups and teardowns
@@ -239,7 +233,6 @@ the default suite name that is got from the file or directory name. New
239233
:option:`--parseinclude` option has been added to `explicitly select which
240234
files are parsed`__ if this kind of parsing optimization is needed.
241235

242-
__ https://github.com/robotframework/robotframework/issues/4721
243236
__ `Selecting files by name or path`_
244237

245238
By tag names
@@ -300,6 +293,28 @@ many interesting possibilities:
300293
the tests for a certain sprint can be generated (for example, `rebot
301294
--include sprint-42 output.xml`).
302295

296+
Options :option:`--include` and :option:`--exclude` can be used in combination
297+
with :option:`--suite` and :option:`--test` discussed in the previous section.
298+
The general rules how they work together are as follows:
299+
300+
- If :option:`--suite` is used, tests must be in the specified suite in addition
301+
to satisfying other selection criteria.
302+
303+
- If :option:`--include` is used with :option:`--test`, it is enough for a test
304+
to match either of them.
305+
306+
- If :option:`--exclude` is used, tests matching it are never selected.
307+
308+
The above rules are demonstrated in the following examples::
309+
310+
--suite example --include tag # Match test if it is in suite 'example' and has tag 'tag'.
311+
--suite example --exclude tag # Match test if it is in suite 'example' and does not have tag 'tag'.
312+
--test example --include tag # Match test if it has name 'example' or it has tag 'tag'.
313+
--test ex* --exclude tag # Match test if its name starts with 'ex' and it does not have tag 'tag'.
314+
315+
.. note:: Prior to Robot Framework 7.0 using `--include` and `--test` together
316+
required test to have both a matching tag and a matching name.
317+
303318
Re-executing failed test cases
304319
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
305320

‎src/robot/model/configurer.py

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,22 +66,26 @@ def _raise_no_tests_or_tasks_error(self, name, rpa):
6666
parts = [{False: 'tests', True: 'tasks', None: 'tests or tasks'}[rpa],
6767
self._get_test_selector_msgs(),
6868
self._get_suite_selector_msg()]
69-
raise DataError("Suite '%s' contains no %s."
70-
% (name, ' '.join(p for p in parts if p)))
69+
raise DataError(f"Suite '{name}' contains no "
70+
f"{' '.join(p for p in parts if p)}.")
7171

7272
def _get_test_selector_msgs(self):
7373
parts = []
74-
for explanation, selector in [('matching tags', self.include_tags),
75-
('not matching tags', self.exclude_tags),
76-
('matching name', self.include_tests)]:
77-
if selector:
78-
parts.append(self._format_selector_msg(explanation, selector))
79-
return seq2str(parts, quote='')
74+
for separator, explanation, selectors in [
75+
(None, 'matching name', self.include_tests),
76+
('or', 'matching tags', self.include_tags),
77+
('and', 'not matching tags', self.exclude_tags)
78+
]:
79+
if selectors:
80+
if parts:
81+
parts.append(separator)
82+
parts.append(self._format_selector_msg(explanation, selectors))
83+
return ' '.join(parts)
8084

81-
def _format_selector_msg(self, explanation, selector):
82-
if len(selector) == 1 and explanation[-1] == 's':
85+
def _format_selector_msg(self, explanation, selectors):
86+
if len(selectors) == 1 and explanation[-1] == 's':
8387
explanation = explanation[:-1]
84-
return '%s %s' % (explanation, seq2str(selector, lastsep=' or '))
88+
return f"{explanation} {seq2str(selectors, lastsep=' or ')}"
8589

8690
def _get_suite_selector_msg(self):
8791
if not self.include_suites:

‎src/robot/model/filter.py

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -81,26 +81,32 @@ def start_suite(self, suite: 'TestSuite'):
8181
if not self:
8282
return False
8383
if hasattr(suite, 'start_time'):
84-
suite.start_time = suite.end_time = None
84+
suite.start_time = suite.end_time = suite.elapsed_time = None
8585
if self.include_suites is not None:
86-
if self.include_suites.match(suite.name, suite.longname):
87-
suite.visit(Filter(include_tests=self.include_tests,
88-
include_tags=self.include_tags,
89-
exclude_tags=self.exclude_tags))
90-
return False
91-
suite.tests = []
92-
return True
93-
if self.include_tests is not None:
94-
suite.tests = [t for t in suite.tests
95-
if self.include_tests.match(t.name, t.longname)]
96-
if self.include_tags is not None:
97-
suite.tests = [t for t in suite.tests
98-
if self.include_tags.match(t.tags)]
99-
if self.exclude_tags is not None:
100-
suite.tests = [t for t in suite.tests
101-
if not self.exclude_tags.match(t.tags)]
86+
return self._filter_based_on_suite_name(suite)
87+
suite.tests = [t for t in suite.tests if self._test_included(t)]
10288
return bool(suite.suites)
10389

90+
def _filter_based_on_suite_name(self, suite: 'TestSuite') -> bool:
91+
if self.include_suites.match(suite.name, suite.longname):
92+
suite.visit(Filter(include_tests=self.include_tests,
93+
include_tags=self.include_tags,
94+
exclude_tags=self.exclude_tags))
95+
return False
96+
suite.tests = []
97+
return True
98+
99+
def _test_included(self, test: 'TestCase') -> bool:
100+
tests, include, exclude \
101+
= self.include_tests, self.include_tags, self.exclude_tags
102+
if exclude is not None and exclude.match(test.tags):
103+
return False
104+
if include is not None and include.match(test.tags):
105+
return True
106+
if tests is not None and tests.match(test.name, test.longname):
107+
return True
108+
return include is None and tests is None
109+
104110
def __bool__(self) -> bool:
105111
return bool(self.include_suites is not None or
106112
self.include_tests is not None or

‎src/robot/model/tags.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ def __add__(self, other: Iterable[str]) -> 'Tags':
105105

106106
class TagPatterns(Sequence['TagPattern']):
107107

108-
def __init__(self, patterns: Iterable[str]):
108+
def __init__(self, patterns: Iterable[str] = ()):
109109
self._patterns = tuple(TagPattern.from_string(p) for p in Tags(patterns))
110110

111111
def match(self, tags: Iterable[str]) -> bool:

‎utest/result/test_configurer.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -89,20 +89,23 @@ def test_no_matching_tests_with_one_selector_each(self):
8989
include_suites='s', include_tests='t')
9090
assert_raises_with_msg(
9191
DataError,
92-
"Suite 'root' contains no tests matching tag 'i', "
93-
"not matching tag 'e' and matching name 't' in suite 's'.",
92+
"Suite 'root' contains no tests matching name 't' "
93+
"or matching tag 'i' "
94+
"and not matching tag 'e' "
95+
"in suite 's'.",
9496
self.suite.visit, configurer
9597
)
9698

9799
def test_no_matching_tests_with_multiple_selectors(self):
98-
configurer = SuiteConfigurer(include_tags=['i1', 'i2'],
100+
configurer = SuiteConfigurer(include_tags=['i1', 'i2', 'i3'],
99101
exclude_tags=['e1', 'e2'],
100102
include_suites=['s1', 's2', 's3'],
101103
include_tests=['t1', 't2'])
102104
assert_raises_with_msg(
103105
DataError,
104-
"Suite 'root' contains no tests matching tags 'i1' or 'i2', "
105-
"not matching tags 'e1' or 'e2' and matching name 't1' or 't2' "
106+
"Suite 'root' contains no tests matching name 't1' or 't2' "
107+
"or matching tags 'i1', 'i2' or 'i3' "
108+
"and not matching tags 'e1' or 'e2' "
106109
"in suites 's1', 's2' or 's3'.",
107110
self.suite.visit, configurer
108111
)

0 commit comments

Comments
 (0)
0