Make sure we are comparing json data structures with each other particuarly in the case of characters outside the BMP where json requires encoding as UTF-16 surrogate pairs. This reinstates the dependency on simplejson which can be lifted if we replicate encoding bits are necessary

jgraham · jgraham · commit b4a73f0c45e4 · 2007-07-15T21:00:27.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40907
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -73,7 +73,7 @@ def normalizeTokens(tokens):
     for i, token in enumerate(tokens):
         if token[0] == u'ParseError':
             tokens[i] = token[0]
-    return tokens
+    return simplejson.loads(simplejson.dumps(tokens))
 
 def tokensMatch(expectedTokens, receivedTokens, ignoreErrorOrder):
     """Test whether the test has passed or failed
