From fefce400ab6ea67b3775cfd682c35eb541662e7d Mon Sep 17 00:00:00 2001 From: Isaac Garzon Date: Sun, 10 Aug 2025 01:26:10 +0300 Subject: [PATCH] tests: actually randomise the order of tests `std::random_shuffle` can use `std::rand` (and does so on Windows), and without intialising with `srand()` the order will always be the same, so we don't get the expected randomisation. Switch to using `std::shuffle` with a proper random number generator in order to fix this and get random order execution. --- test/harness.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/harness.cpp b/test/harness.cpp index 2eee03738..ef14799df 100644 --- a/test/harness.cpp +++ b/test/harness.cpp @@ -3,6 +3,8 @@ // // Copyright 2016 whitequark //----------------------------------------------------------------------------- +#include +#include #include #include @@ -337,19 +339,29 @@ int Test::Case::Register(Test::Case testCase) { int main(int argc, char **argv) { std::vector args = Platform::InitCli(argc, argv); - std::regex filter(".*"); + std::string filterPattern = ".*"; + unsigned int seed = std::random_device{}(); if(args.size() == 1) { } else if(args.size() == 2) { - filter = args[1]; + filterPattern = args[1]; + } else if(args.size() == 3) { + filterPattern = args[1]; + seed = std::stoul(args[2]); } else { - fprintf(stderr, "Usage: %s [test filter regex]\n", args[0].c_str()); + fprintf(stderr, "Usage: %s [test filter regex] [shuffle seed]\n", args[0].c_str()); return 1; } + fprintf(stderr, "info: using test filter `%s' and seed %u\n", filterPattern.c_str(), seed); + Platform::fontFiles.push_back(HostRoot().Join("Gentium-R.ttf")); + std::mt19937 g(seed); + // Wreck order dependencies between tests! - std::random_shuffle(testCasesPtr->begin(), testCasesPtr->end()); + std::shuffle(testCasesPtr->begin(), testCasesPtr->end(), g); + + std::regex filter(filterPattern); auto testStartTime = std::chrono::steady_clock::now(); size_t ranTally = 0, skippedTally = 0, checkTally = 0, failTally = 0;