refactor image test scripts:

- init test/image/assets/ - move get_image_request_option to assets/ - move image test folders init to pretest.js - factor out get image path logic in script into asset file - add + centralize glob pattern matching - add --queue option for compare pixel script - run baseline generation in queue (instead of in batch)
plotly · etpinard · Jun 20, 2016 · Jun 17, 2016 · Jun 17, 2016 · Jun 17, 2016
commit 66bf29eaded9e203b21b8219a03a2f3b72c71ba8
diff --git a/tasks/pretest.js b/tasks/pretest.js
@@ -36,3 +36,11 @@ var setPlotConfig = [
 fs.writeFile(constants.pathToSetPlotConfig, setPlotConfig, function(err) {
     if(err) throw err;
 });
+
+// make artifact folders for image tests
+if(!fs.existsSync(constants.pathToTestImagesDiff)) {
+    fs.mkdirSync(constants.pathToTestImagesDiff);
+}
+if(!fs.existsSync(constants.pathToTestImages)) {
+    fs.mkdirSync(constants.pathToTestImages);
+}
diff --git a/tasks/util/get_image_request_options.js b/tasks/util/get_image_request_options.js
diff --git a/test/image/assets/get_image_paths.js b/test/image/assets/get_image_paths.js
@@ -0,0 +1,29 @@
+var path = require('path');
+var constants = require('../../../tasks/util/constants');
+
+var DEFAULT_FORMAT = 'png';
+
+
+/**
+ * Return paths to baseline, test-image and diff images for a given mock name.
+ *
+ * @param {string} mockName
+ * @param {string} format
+ * @return {object}
+ *      baseline
+ *      test
+ *      diff
+ */
+module.exports = function getImagePaths(mockName, format) {
+    format = format || DEFAULT_FORMAT;
+
+    return {
+        baseline: join(constants.pathToTestImageBaselines, mockName, format),
+        test: join(constants.pathToTestImages, mockName, format),
+        diff: join(constants.pathToTestImagesDiff, mockName, format)
+    };
+};
+
+function join(basePath, mockName, format) {
+    return path.join(basePath, mockName) + '.' + format;
+}
diff --git a/test/image/assets/get_image_request_options.js b/test/image/assets/get_image_request_options.js
@@ -0,0 +1,32 @@
+var path = require('path');
+var constants = require('../../../tasks/util/constants');
+
+var DEFAULT_URL = 'http://localhost:9010/';
+var DEFAULT_FORMAT = 'png';
+var DEFAULT_SCALE = 1;
+
+/**
+ *  Return the image server request options for a given mock (and specs)
+ *
+ *  @param {object} specs
+ *      mockName : name of json mock to plot
+ *      format (optional): format of generated image
+ *      scale (optional): scale of generated image
+ *      url (optional): URL of image server
+ */
+module.exports = function getRequestOpts(specs) {
+    var pathToMock = path.join(constants.pathToTestImageMocks, specs.mockName) + '.json';
+    var figure = require(pathToMock);
+
+    var body = {
+        figure: figure,
+        format: specs.format || DEFAULT_FORMAT,
+        scale: specs.scale || DEFAULT_SCALE
+    };
+
+    return {
+        method: 'POST',
+        url: specs.url || DEFAULT_URL,
+        body: JSON.stringify(body)
+    };
+};
diff --git a/test/image/assets/get_mock_list.js b/test/image/assets/get_mock_list.js
@@ -0,0 +1,29 @@
+var path = require('path');
+var glob = require('glob');
+
+var constants = require('../../../tasks/util/constants');
+
+
+/**
+ *  Return array of mock name corresponding to input glob pattern
+ *
+ *  @param {string} pattern
+ *  @return {array}
+ */
+module.exports = function getMocks(pattern) {
+    // defaults to 'all'
+    pattern = pattern || '*';
+
+    // defaults to '.json' ext is none is provided
+    if(path.extname(pattern) === '') pattern += '.json';
+
+    var patternFull = constants.pathToTestImageMocks + '/' + pattern;
+    var matches = glob.sync(patternFull);
+
+    // return only the mock name (not a full path, no ext)
+    var mockNames = matches.map(function(match) {
+        return path.basename(match).split('.')[0];
+    });
+
+    return mockNames;
+};
diff --git a/test/image/compare_pixels_test.js b/test/image/compare_pixels_test.js
@@ -1,109 +1,156 @@
 var fs = require('fs');
-var path = require('path');
 
-var constants = require('../../tasks/util/constants');
-var getOptions = require('../../tasks/util/get_image_request_options');
+var getMockList = require('./assets/get_mock_list');
+var getRequestOpts = require('./assets/get_image_request_options');
+var getImagePaths = require('./assets/get_image_paths');
 
 // packages inside the image server docker
-var request = require('request');
 var test = require('tape');
+var request = require('request');
 var gm = require('gm');
 
-var TOLERANCE = 1e-6;    // pixel comparison tolerance
-var BASE_TIMEOUT = 500;  // base timeout time
-var BATCH_SIZE = 5;      // size of each test 'batch'
-var running = 0;         // number of tests currently running
+// pixel comparison tolerance
+var TOLERANCE = 1e-6;
+
+// wait time between each test batch
+var BATCH_WAIT = 500;
+
+// number of tests in each test batch
+var BATCH_SIZE = 5;
+
+// wait time between each test in test queue
+var QUEUE_WAIT = 10;
+
+/**
+ *  Image pixel comparison test script.
+ *
+ *  Called by `tasks/test_image.sh in `npm run test-image`.
+ *
+ *  CLI arguments:
+ *
+ *  1. 'pattern' :
+ *      glob determining which mock(s) are to be tested
+ *  2. --queue :
+ *      if sent, the image will be run in queue instead of in batch.
+ *      Makes the test run significantly longer, but is Recommended on weak hardware.
+ *
+ *  Examples:
+ *
+ *  Run all tests in batch:
+ *
+ *      npm run test-image
+ *
+ *  Run the 'contour_nolines' test:
+ *
+ *      npm run test-image -- contour_nolines
+ *
+ *  Run all gl3d image test in queue:
+ *
+ *      npm run test-image -- gl3d_* --queue
+ */
+
+var pattern = process.argv[2];
+var mockList = getMockList(pattern);
+var isInQueue = (process.argv[3] === '--queue');
+
+if(mockList.length === 0) {
+    throw new Error('No mocks found with pattern ' + pattern);
+}
 
-var touch = function(fileName) {
-    fs.closeSync(fs.openSync(fileName, 'w'));
-};
+mockList = mockList.filter(untestableFilter);
 
+if(mockList.length === 0) {
+    throw new Error('All mocks found with pattern ' + pattern + ' are currently untestable');
+}
 
-// make artifact folders
-if(!fs.existsSync(constants.pathToTestImagesDiff)) {
-    fs.mkdirSync(constants.pathToTestImagesDiff);
+// main
+if(isInQueue) {
+    runInQueue(mockList);
 }
-if(!fs.existsSync(constants.pathToTestImages)) {
-    fs.mkdirSync(constants.pathToTestImages);
+else {
+    runInBatch(mockList);
 }
 
-var userFileName = process.argv[2];
+/* Test cases:
+ *
+ * - font-wishlist
+ * - all gl2d
+ * - all mapbox
+ *
+ * don't behave consistently from run-to-run and/or
+ * machine-to-machine; skip over them for now.
+ *
+ */
+function untestableFilter(mockName) {
+    return !(
+        mockName === 'font-wishlist' ||
+        mockName.indexOf('gl2d_') !== -1 ||
+        mockName.indexOf('mapbox_') !== -1
+    );
+}
 
-// run the test(s)
-if(!userFileName) runAll();
-else runSingle(userFileName);
+function runInBatch(mockList) {
+    var running = 0;
 
-function runAll() {
-    test('testing mocks', function(t) {
+    // remove mapbox mocks if circle ci
 
-        var fileNames = fs.readdirSync(constants.pathToTestImageMocks);
+    test('testing mocks in batch', function(t) {
+        t.plan(mockList.length);
 
-        // eliminate pollutants (e.g .DS_Store) that can accumulate in the mock directory
-        var allMocks = fileNames.filter(function(name) {return name.slice(-5) === '.json';});
+        for(var i = 0; i < mockList.length; i++) {
+            run(mockList[i], t);
+        }
+    });
 
-        /* Test cases:
-         *
-         * - font-wishlist
-         * - all gl2d
-         *
-         * don't behave consistently from run-to-run and/or
-         * machine-to-machine; skip over them.
-         *
-         */
-        var mocks = allMocks.filter(function(mock) {
-            return !(
-                mock === 'font-wishlist.json' ||
-                mock.indexOf('gl2d') !== -1
-            );
-        });
+    function run(mockName, t) {
+        if(running >= BATCH_SIZE) {
+            setTimeout(function() {
+                run(mockName, t);
+            }, BATCH_WAIT);
+            return;
+        }
+        running++;
 
-        // skip mapbox mocks for now
-        mocks = mocks.filter(function(mock) {
-            return mock.indexOf('mapbox_
F438
9;) === -1;
+        // throttle the number of tests running concurrently
+
+        comparePixels(mockName, function(isEqual, mockName) {
+            running--;
+            t.ok(isEqual, mockName + ' should be pixel perfect');
         });
+    }
+}
 
-        t.plan(mocks.length);
+function runInQueue(mockList) {
+    var index = 0;
 
-        for(var i = 0; i < mocks.length; i++) {
-            testMock(mocks[i], t);
-        }
+    test('testing mocks in queue', function(t) {
+        t.plan(mockList.length);
 
+        run(mockList[index], t);
     });
-}
 
-function runSingle(userFileName) {
-    test('testing single mock: ' + userFileName, function(t) {
-        t.plan(1);
-        testMock(userFileName, t);
-    });
-}
+    function run(mockName, t) {
+        comparePixels(mockName, function(isEqual, mockName) {
+            t.ok(isEqual, mockName + ' should be pixel perfect');
 
-function testMock(fileName, t) {
-    // throttle the number of tests running concurrently
-    if(running >= BATCH_SIZE) {
-        setTimeout(function() { testMock(fileName, t); }, BASE_TIMEOUT);
-        return;
+            index++;
+            if(index < mockList.length) {
+                setTimeout(function() {
+                    run(mockList[index], t);
+                }, QUEUE_WAIT);
+            }
+        });
     }
-    running++;
-
-    var figure = require(path.join(constants.pathToTestImageMocks, fileName));
-    var bodyMock = {
-        figure: figure,
-        format: 'png',
-        scale: 1
-    };
+}
 
-    var imageFileName = fileName.split('.')[0] + '.png';
-    var savedImagePath = path.join(constants.pathToTestImages, imageFileName);
-    var diffPath = path.join(constants.pathToTestImagesDiff, 'diff-' + imageFileName);
-    var savedImageStream = fs.createWriteStream(savedImagePath);
-    var options = getOptions(bodyMock, 'http://localhost:9010/');
+function comparePixels(mockName, cb) {
+    var requestOpts = getRequestOpts({ mockName: mockName }),
+        imagePaths = getImagePaths(mockName),
+        saveImageStream = fs.createWriteStream(imagePaths.test);
 
     function checkImage() {
-        running--;
-
-        var options = {
-            file: diffPath,
+        var gmOpts = {
+            file: imagePaths.diff,
             highlightColor: 'purple',
             tolerance: TOLERANCE
         };
@@ -125,26 +172,30 @@ function testMock(fileName, t) {
          */
 
         gm.compare(
-            savedImagePath,
-            path.join(constants.pathToTestImageBaselines, imageFileName),
-            options,
+            imagePaths.test,
+            imagePaths.baseline,
+            gmOpts,
             onEqualityCheck
         );
     }
 
     function onEqualityCheck(err, isEqual) {
         if(err) {
-            touch(diffPath);
-            return console.error(err, imageFileName);
+            touch(imagePaths.diff);
+            return console.error(err, mockName);
         }
         if(isEqual) {
-            fs.unlinkSync(diffPath);
+            fs.unlinkSync(imagePaths.diff);
         }
 
-        t.ok(isEqual, imageFileName + ' should be pixel perfect');
+        cb(isEqual, mockName);
     }
 
-    request(options)
-        .pipe(savedImageStream)
+    request(requestOpts)
+        .pipe(saveImageStream)
         .on('close', checkImage);
 }
+
+function touch(filePath) {
+    fs.closeSync(fs.openSync(filePath, 'w'));
+}