8000 Refactor ordering of tests · ruby/ruby@c4570ac · GitHub
[go: up one dir, main page]
Skip to content
8000

Commit c4570ac

Browse files
nobunobu
committed
Refactor ordering of tests
* Split the sorting types into classes. * Apply the same sorting to method sorting under the parallel test.
1 parent 44b2e32 commit c4570ac

File tree

4 files changed

+157
-78
lines changed

4 files changed

+157
-78
lines changed

‎tool/lib/test/unit.rb

Lines changed: 94 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,84 @@ class AssertionFailedError < Exception; end
5757

5858
class PendedError < AssertionFailedError; end
5959

60+
module Order
61+
class NoSort
62+
def initialize(seed)
63+
end
64+
65+
def sort_by_name(list)
66+
list
67+
end
68+
69+
alias sort_by_string sort_by_name
70+
71+
def group(list)
72+
# JIT first
73+
jit, others = list.partition {|e| /test_jit/ =~ e}
74+
jit + others
75+
end
76+
end
77+
78+
class Alpha < NoSort
79+
def sort_by_name(list)
80+
list.sort_by(&:name)
81+
end
82+
83+
def sort_by_string(list)
84+
list.sort
85+
end
86+
87+
end
88+
89+
# shuffle test suites based on CRC32 of their names
90+
Shuffle = Struct.new(:seed, :salt) do
91+
def initialize(seed)
92+
self.class::CRC_TBL ||= (0..255).map {|i|
93+
(0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) }
94+
}.freeze
95+
96+
salt = [seed].pack("V").unpack1("H*")
97+
super(seed, "\n#{salt}".freeze).freeze
98+
end
99+
100+
def sort_by_name(list)
101+
list.sort_by {|e| randomize_key(e.name)}
102+
end
103+
104+
def sort_by_string(list)
105+
list.sort_by {|e| randomize_key(e)}
106+
end
107+
108+
def group(list)
109+
list
110+
end
111+
112+
private
113+
114+
def crc32(str, crc32 = 0xffffffff)
115+
crc_tbl = self.class::CRC_TBL
116+
str.each_byte do |data|
117+
crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8)
118+
end
119+
crc32
120+
end
121+
122+
def randomize_key(name)
123+
crc32(salt, crc32(name)) ^ 0xffffffff
124+
end
125+
end
126+
127+
Types = {
128+
random: Shuffle,
129+
alpha: Alpha,
130+
sorted: Alpha,
131+
nosort: NoSort,
132+
}
133+
Types.default_proc = proc {|_, order|
134+
raise "Unknown test_order: #{order.inspect}"
135+
}
136+
end
137+
60138
module RunCount # :nodoc: all
61139
@@run_count = 0
62140

@@ -103,13 +181,13 @@ def process_args(args = [])
103181
order = options[:test_order]
104182
if seed = options[:seed]
105183
order ||= :random
106-
srand(seed)
107-
else
108-
seed = options[:seed] = srand % 100_000
109-
srand(seed)
184+
elsif order == :random
185+
seed = options[:seed] = rand(0x10000)
110186
orig_args.unshift "--seed=#{seed}"
111187
end
112188
Test::Unit::TestCase.test_order = order if order
189+
order = Test::Unit::TestCase.test_order
190+
@order = Test::Unit::Order::Types[order].new(seed)
113191

114192
@help = "\n" + orig_args.map { |s|
115193
" " + (s =~ /[\s|&<>$()]/ ? s.inspect : s)
@@ -139,7 +217,8 @@ def setup_options(opts, options)
139217
(options[:filter] ||= []) << a
140218
end
141219

142-
opts.on '--test-order=random|alpha|sorted|nosort', [:random, :alpha, :sorted, :nosort] do |a|
220+
orders = Test::Unit::Order::Types.keys
221+
opts.on "--test-order=#{orders.join('|')}", orders do |a|
143222
options[:test_order] = a
144223
end
145224
end
@@ -545,16 +624,7 @@ def _run_parallel suites, type, result
545624

546625
# Require needed thing for parallel running
547626
require 'timeout'
548-
@tasks = @files.dup # Array of filenames.
549-
550-
case Test::Unit::TestCase.test_order
551-
when :random
552-
@tasks.shuffle!
553-
else
554-
# JIT first
555-
ts = @tasks.group_by{|e| /test_jit/ =~ e ? 0 : 1}
556-
@tasks = ts[0] + ts[1] if ts.size == 2
557-
end
627+
@tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames.
558628

559629
@need_quit = false
560630
@dead_workers = [] # Array of dead workers.
@@ -1302,6 +1372,8 @@ def _run_anything type
13021372
suites = Test::Unit::TestCase.send "#{type}_suites"
13031373
return if suites.empty?
13041374

1375+
suites = @order.sort_by_name(suites)
1376+
13051377
puts
13061378
puts "# Running #{type}s:"
13071379
puts
@@ -1356,17 +1428,20 @@ def _run_suite suite, type
13561428
filter = options[:filter]
13571429

13581430
all_test_methods = suite.send "#{type}_methods"
1431+
if filter
1432+
all_test_methods.select! {|method|
1433+
filter === method || filter === "#{suite}##{method}"
1434+
}
1435+
end
1436+
all_test_methods = @order.sort_by_name(all_test_methods)
13591437

13601438
leakchecker = LeakChecker.new
13611439
if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"]
13621440
require "objspace"
13631441
trace = true
13641442
end
13651443

1366-
assertions = all_test_methods.filter_map { |method|
1367-
if filter
1368-
next unless filter === method || filter === "#{suite}##{method}"
1369-
end
1444+
assertions = all_test_methods.map { |method|
13701445

13711446
inst = suite.new method
13721447
inst._assertions = 0

‎tool/lib/test/unit/testcase.rb

Lines changed: 2 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,6 @@ def run runner
159159
start_time = Time.now
160160

161161
result = ""
162-
srand(runner.options[:seed])
163162

164163
begin
165164
@passed = nil
@@ -267,46 +266,11 @@ def self.test_order
267266
end
268267

269268
def self.test_suites # :nodoc:
270-
suites = @@test_suites.keys
271-
272-
case self.test_order
273-
when :random
274-
# shuffle test suites based on CRC32 of their names
275-
salt = "\n" + rand(1 << 32).to_s
276-
crc_tbl = (0..255).map do |i|
277-
(0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) }
278-
end
279-
suites = suites.sort_by do |suite|
280-
crc32 = 0xffffffff
281-
"#{suite.name}#{salt}".each_byte do |data|
282-
crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8)
283-
end
284-
crc32 ^ 0xffffffff
285-
end
286-
when :nosort
287-
suites
288-
else
289-
suites.sort_by { |ts| ts.name.to_s }
290-
end
269+
@@test_suites.keys
291270
end
292271

293272
def self.test_methods # :nodoc:
294-
methods = public_instance_methods(true).grep(/^test/).map { |m| m.to_s }
295-
296-
case self.test_order
297-
when :parallel
298-
max = methods.size
299-
ParallelEach.new methods.sort.sort_by { rand max }
300-
when :random then
301-
max = methods.size
302-
methods.sort.sort_by { rand max }
303-
when :alpha, :sorted then
304-
methods.sort
305-
when :nosort
306-
methods
307-
else
308-
raise "Unknown test_order: #{self.test_order.inspect}"
309-
end
273+
public_instance_methods(true).grep(/^test/)
310274
end
311275

312276
##

‎tool/test/testunit/test_minitest_unit.rb

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ def test_class_test_suites
238238
tc = Class.new(Test::Unit::TestCase)
239239

240240
assert_equal 2, Test::Unit::TestCase.test_suites.size
241-
assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites
241+
assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites.sort_by {|ts| ts.name.to_s}
242242
end
243243

244244
def assert_filtering name, expected, a = false
@@ -1331,34 +1331,17 @@ def test_skip
13311331
end
13321332
end
13331333

1334-
def test_test_methods_random
1334+
def test_test_methods
13351335
@assertion_count = 0
13361336

13371337
sample_test_case = Class.new Test::Unit::TestCase do
1338-
def self.test_order; :random; end
13391338
def test_test1; assert "does not matter" end
13401339
def test_test2; assert "does not matter" end
13411340
def test_test3; assert "does not matter" end
1342-
@test_order = [1, 0, 2]
1343-
def self.rand(n) @test_order.shift; end
13441341
end
13451342

1346-
expected = %w(test_test2 test_test1 test_test3)
1347-
assert_equal expected, sample_test_case.test_methods
1348-
end
1349-
1350-
def test_test_methods_sorted
1351-
@assertion_count = 0
1352-
1353-
sample_test_case = Class.new Test::Unit::TestCase do
1354-
def self.test_order; :sorted end
1355-
def test_test3; assert "does not matter" end
1356-
def test_test2; assert "does not matter" end
1357-
def test_test1; assert "does not matter" end
1358-
end
1359-
1360-
expected = %w(test_test1 test_test2 test_test3)
1361-
assert_equal expected, sample_test_case.test_methods
1343+
expected = %i(test_test1 test_test2 test_test3)
1344+
assert_equal expected, sample_test_case.test_methods.sort
13621345
end
13631346

13641347
def assert_triggered expected, klass = Test::Unit::AssertionFailedError

‎tool/test/testunit/test_sorting.rb

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,61 @@ def sorting(*args)
1515
f.read
1616
}
1717
end
18+
19+
Item = Struct.new(:name)
20+
SEED = 0x50975eed
21+
22+
def make_test_list
23+
(1..16).map {"test_%.3x" % rand(0x1000)}.freeze
24+
end
25+
26+
def test_sort_alpha
27+
sorter = Test::Unit::Order::Types[:alpha].new(SEED)
28+
assert_kind_of(Test::Unit::Order::Types[:sorted], sorter)
29+
30+
list = make_test_list
31+
sorted = list.sort
32+
16.times do
33+
assert_equal(sorted, sorter.sort_by_string(list))
34+
end
35+
36+
list = list.map {|s| Item.new(s)}.freeze
37+
sorted = list.sort_by(&:name)
38+
16.times do
39+
assert_equal(sorted, sorter.sort_by_name(list))
40+
end
41+
end
42+
43+
def test_sort_nosort
44+
sorter = Test::Unit::Order::Types[:nosort].new(SEED)
45+
46+
list = make_test_list
47+
16.times do
48+
assert_equal(list, sorter.sort_by_string(list))
49+
end
50+
51+
list = list.map {|s| Item.new(s)}.freeze
52+
16.times do
53+
assert_equal(list, sorter.sort_by_name(list))
54+
end
55+
end
56+
57+
def test_sort_random
58+
type = Test::Unit::Order::Types[:random]
59+
sorter = type.new(SEED)
60+
61+
list = make_test_list
62+
sorted = type.new(SEED).sort_by_string(list).freeze
63+
16.times do
64+
assert_equal(sorted, sorter.sort_by_string(list))
65+
end
66+
assert_not_equal(sorted, type.new(SEED+1).sort_by_string(list))
67+
68+
list = list.map {|s| Item.new(s)}.freeze
69+
sorted = sorted.map {|s| Item.new(s)}.freeze
70+
16.times do
71+
assert_equal(sorted, sorter.sort_by_name(list))
72+
end
73+
assert_not_equal(sorted, type.new(SEED+1).sort_by_name(list))
74+
end
1875
end

0 commit comments

Comments
 (0)
0