From 5705247e5f16d5aa2b14b306f3c1de8c400b3024 Mon Sep 17 00:00:00 2001 From: Benoit Blanchon Date: Thu, 4 Aug 2022 10:47:02 +0200 Subject: [PATCH 001/152] Fix replacement for `getOrAddElement()` and `getOrAddMember()` --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe78ab06f..3cf0f670d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,9 @@ HEAD > > // after > JsonVariant a = variant[idx]; -> JsonVariant b = variant[idx].to(); +> JsonVariant b = idx < variant.size() ? variant[idx] : variant[idx].to(); > JsonVariant c = variant[key]; -> JsonVariant d = variant[key].to(); +> JsonVariant d = variant.containsKey(key) ? variant[key] : variant[key].to(); > ``` v6.19.4 (2022-04-05) From 1d21027e2a0fa714e37418add40aab7808fcf66b Mon Sep 17 00:00:00 2001 From: Benoit Blanchon Date: Thu, 4 Aug 2022 12:39:34 +0200 Subject: [PATCH 002/152] Fix lax parsing of `true`, `false`, and `null` (fixes #1781) --- CHANGELOG.md | 1 + extras/tests/JsonDeserializer/filter.cpp | 42 +++++++- .../tests/JsonDeserializer/invalid_input.cpp | 6 +- src/ArduinoJson/Json/JsonDeserializer.hpp | 101 +++++++++++++----- 4 files changed, 116 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cf0f670d..5c89e2a4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ HEAD * Add `JsonVariant::shallowCopy()` (issue #1343) * Fix `9.22337e+18 is outside the range of representable values of type 'long'` * Fix comparison operators for `JsonArray`, `JsonArrayConst`, `JsonObject`, and `JsonObjectConst` +* Fix lax parsing of `true`, `false`, and `null` (issue #1781) * Remove undocumented `accept()` functions * Rename `addElement()` to `add()` * Remove `getElement()`, `getOrAddElement()`, `getMember()`, and `getOrAddMember()` diff --git a/extras/tests/JsonDeserializer/filter.cpp b/extras/tests/JsonDeserializer/filter.cpp index 9cf85d27a..e0c373410 100644 --- a/extras/tests/JsonDeserializer/filter.cpp +++ b/extras/tests/JsonDeserializer/filter.cpp @@ -71,6 +71,15 @@ TEST_CASE("Filtering") { "{\"example\":null}", JSON_OBJECT_SIZE(1) + 8 }, + { + // Member is a number, but filter wants an array + "{\"example\":42}", + "{\"example\":[true]}", + 10, + DeserializationError::Ok, + "{\"example\":null}", + JSON_OBJECT_SIZE(1) + 8 + }, { // Input is an array, but filter wants an object "[\"hello\",\"world\"]", @@ -117,7 +126,7 @@ TEST_CASE("Filtering") { JSON_OBJECT_SIZE(1) + 8 }, { - // can skip a boolean + // skip false "{\"a_bool\":false,example:42}", "{\"example\":true}", 10, @@ -125,6 +134,24 @@ TEST_CASE("Filtering") { "{\"example\":42}", JSON_OBJECT_SIZE(1) + 8 }, + { + // skip true + "{\"a_bool\":true,example:42}", + "{\"example\":true}", + 10, + DeserializationError::Ok, + "{\"example\":42}", + JSON_OBJECT_SIZE(1) + 8 + }, + { + // skip null + "{\"a_bool\":null,example:42}", + "{\"example\":true}", + 10, + DeserializationError::Ok, + "{\"example\":42}", + JSON_OBJECT_SIZE(1) + 8 + }, { // can skip a double-quoted string "{\"a_double_quoted_string\":\"hello\",example:42}", @@ -618,7 +645,7 @@ TEST_CASE("Filtering") { 0 }, { - // incomplete after after key of a skipped object + // incomplete comment after key of a skipped object "{\"example\"/*:2}", "false", 10, @@ -636,7 +663,7 @@ TEST_CASE("Filtering") { 0 }, { - // incomplete after after value of a skipped object + // incomplete comment after value of a skipped object "{\"example\":2/*}", "false", 10, @@ -644,6 +671,15 @@ TEST_CASE("Filtering") { "null", 0 }, + { + // incomplete comment after comma in skipped object + "{\"example\":2,/*}", + "false", + 10, + DeserializationError::IncompleteInput, + "null", + 0 + }, }; // clang-format on for (size_t i = 0; i < sizeof(testCases) / sizeof(testCases[0]); i++) { diff --git a/extras/tests/JsonDeserializer/invalid_input.cpp b/extras/tests/JsonDeserializer/invalid_input.cpp index f35800862..a2304a372 100644 --- a/extras/tests/JsonDeserializer/invalid_input.cpp +++ b/extras/tests/JsonDeserializer/invalid_input.cpp @@ -9,7 +9,8 @@ TEST_CASE("Invalid JSON input") { const char* testCases[] = {"'\\u'", "'\\u000g'", "'\\u000'", "'\\u000G'", "'\\u000/'", "\\x1234", "6a9", "1,", - "2]", "3}"}; + "nulL", "tru3", "fals3", "2]", + "3}"}; const size_t testCount = sizeof(testCases) / sizeof(testCases[0]); DynamicJsonDocument doc(4096); @@ -23,9 +24,6 @@ TEST_CASE("Invalid JSON input") { TEST_CASE("Invalid JSON input that should pass") { const char* testCases[] = { - "nulL", - "tru3", - "fals3", "'\\ud83d'", // leading surrogate without a trailing surrogate "'\\udda4'", // trailing surrogate without a leading surrogate "'\\ud83d\\ud83d'", // two leading surrogates diff --git a/src/ArduinoJson/Json/JsonDeserializer.hpp b/src/ArduinoJson/Json/JsonDeserializer.hpp index 569e65796..90e5951c9 100644 --- a/src/ArduinoJson/Json/JsonDeserializer.hpp +++ b/src/ArduinoJson/Json/JsonDeserializer.hpp @@ -85,7 +85,22 @@ class JsonDeserializer { if (filter.allowValue()) return parseStringValue(variant); else - return skipString(); + return skipQuotedString(); + + case 't': + if (filter.allowValue()) + variant.setBoolean(true); + return skipKeyword("true"); + + case 'f': + if (filter.allowValue()) + variant.setBoolean(false); + return skipKeyword("false"); + + case 'n': + // the variant should already by null, except if the same object key was + // used twice, as in {"a":1,"a":null} + return skipKeyword("null"); default: if (filter.allowValue()) @@ -111,7 +126,16 @@ class JsonDeserializer { case '\"': case '\'': - return skipString(); + return skipQuotedString(); + + case 't': + return skipKeyword("true"); + + case 'f': + return skipKeyword("false"); + + case 'n': + return skipKeyword("null"); default: return skipNumericValue(); @@ -310,7 +334,7 @@ class JsonDeserializer { // Read each key value pair for (;;) { // Skip key - err = skipVariant(nestingLimit.decrement()); + err = skipKey(); if (err) return err; @@ -338,6 +362,10 @@ class JsonDeserializer { return DeserializationError::Ok; if (!eat(',')) return DeserializationError::InvalidInput; + + err = skipSpacesAndComments(); + if (err) + return err; } } @@ -438,7 +466,15 @@ class JsonDeserializer { return DeserializationError::Ok; } - DeserializationError::Code skipString() { + DeserializationError::Code skipKey() { + if (isQuote(current())) { + return skipQuotedString(); + } else { + return skipNonQuotedString(); + } + } + + DeserializationError::Code skipQuotedString() { const char stopChar = current(); move(); @@ -458,37 +494,26 @@ class JsonDeserializer { return DeserializationError::Ok; } + DeserializationError::Code skipNonQuotedString() { + char c = current(); + while (canBeInNonQuotedString(c)) { + move(); + c = current(); + } + return DeserializationError::Ok; + } + DeserializationError::Code parseNumericValue(VariantData &result) { uint8_t n = 0; char c = current(); - while (canBeInNonQuotedString(c) && n < 63) { + while (canBeInNumber(c) && n < 63) { move(); _buffer[n++] = c; c = current(); } _buffer[n] = 0; - c = _buffer[0]; - if (c == 't') { // true - result.setBoolean(true); - if (n != 4) - return DeserializationError::IncompleteInput; - return DeserializationError::Ok; - } - if (c == 'f') { // false - result.setBoolean(false); - if (n != 5) - return DeserializationError::IncompleteInput; - return DeserializationError::Ok; - } - if (c == 'n') { // null - // the variant is already null - if (n != 4) - return DeserializationError::IncompleteInput; - return DeserializationError::Ok; - } - if (!parseNumber(_buffer, result)) return DeserializationError::InvalidInput; @@ -497,7 +522,7 @@ class JsonDeserializer { DeserializationError::Code skipNumericValue() { char c = current(); - while (canBeInNonQuotedString(c)) { + while (canBeInNumber(c)) { move(); c = current(); } @@ -523,9 +548,18 @@ class JsonDeserializer { return min <= c && c <= max; } + static inline bool canBeInNumber(char c) { + return isBetween(c, '0', '9') || c == '+' || c == '-' || c == '.' || +#if ARDUINOJSON_ENABLE_NAN || ARDUINOJSON_ENABLE_INFINITY + isBetween(c, 'A', 'Z') || isBetween(c, 'a', 'z'); +#else + c == 'e' || c == 'E'; +#endif + } + static inline bool canBeInNonQuotedString(char c) { return isBetween(c, '0', '9') || isBetween(c, '_', 'z') || - isBetween(c, 'A', 'Z') || c == '+' || c == '-' || c == '.'; + isBetween(c, 'A', 'Z'); } static inline bool isQuote(char c) { @@ -605,6 +639,19 @@ class JsonDeserializer { } } + DeserializationError::Code skipKeyword(const char *s) { + while (*s) { + char c = current(); + if (c == '\0') + return DeserializationError::IncompleteInput; + if (*s != c) + return DeserializationError::InvalidInput; + ++s; + move(); + } + return DeserializationError::Ok; + } + TStringStorage _stringStorage; bool _foundSomething; Latch _latch; From 62e83133cd4d82deac43bce461b950b4e609ad15 Mon Sep 17 00:00:00 2001 From: Benoit Blanchon Date: Tue, 9 Aug 2022 10:33:19 +0200 Subject: [PATCH 003/152] Remove `JsonDocument::data()` and `JsonDocument::memoryPool()` --- CHANGELOG.md | 1 + extras/tests/MemoryPool/StringCopier.cpp | 10 +++---- extras/tests/Misc/Utf8.cpp | 2 +- .../Deserialization/deserialize.hpp | 30 +++++++++++-------- .../Document/BasicJsonDocument.hpp | 2 +- src/ArduinoJson/Document/JsonDocument.hpp | 10 ------- src/ArduinoJson/Json/JsonDeserializer.hpp | 4 +-- .../MsgPack/MsgPackDeserializer.hpp | 4 +-- .../StringStorage/StringCopier.hpp | 2 +- .../StringStorage/StringStorage.hpp | 5 ++-- 10 files changed, 33 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c89e2a4e..673975f85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ HEAD * Remove undocumented `accept()` functions * Rename `addElement()` to `add()` * Remove `getElement()`, `getOrAddElement()`, `getMember()`, and `getOrAddMember()` +* Remove `JsonDocument::data()` and `JsonDocument::memoryPool()` > ### BREAKING CHANGES > diff --git a/extras/tests/MemoryPool/StringCopier.cpp b/extras/tests/MemoryPool/StringCopier.cpp index ba39cfd98..c6159053f 100644 --- a/extras/tests/MemoryPool/StringCopier.cpp +++ b/extras/tests/MemoryPool/StringCopier.cpp @@ -12,7 +12,7 @@ TEST_CASE("StringCopier") { SECTION("Works when buffer is big enough") { MemoryPool pool(buffer, addPadding(JSON_STRING_SIZE(5))); - StringCopier str(pool); + StringCopier str(&pool); str.startString(); str.append("hello"); @@ -24,7 +24,7 @@ TEST_CASE("StringCopier") { SECTION("Returns null when too small") { MemoryPool pool(buffer, sizeof(void*)); - StringCopier str(pool); + StringCopier str(&pool); str.startString(); str.append("hello world!"); @@ -35,7 +35,7 @@ TEST_CASE("StringCopier") { SECTION("Increases size of memory pool") { MemoryPool pool(buffer, addPadding(JSON_STRING_SIZE(6))); - StringCopier str(pool); + StringCopier str(&pool); str.startString(); str.save(); @@ -46,7 +46,7 @@ TEST_CASE("StringCopier") { SECTION("Works when memory pool is 0 bytes") { MemoryPool pool(buffer, 0); - StringCopier str(pool); + StringCopier str(&pool); str.startString(); REQUIRE(str.isValid() == false); @@ -55,7 +55,7 @@ TEST_CASE("StringCopier") { } static const char* addStringToPool(MemoryPool& pool, const char* s) { - StringCopier str(pool); + StringCopier str(&pool); str.startString(); str.append(s); return str.save().c_str(); diff --git a/extras/tests/Misc/Utf8.cpp b/extras/tests/Misc/Utf8.cpp index 38e839ac3..b44028ae3 100644 --- a/extras/tests/Misc/Utf8.cpp +++ b/extras/tests/Misc/Utf8.cpp @@ -12,7 +12,7 @@ using namespace ARDUINOJSON_NAMESPACE; static void testCodepoint(uint32_t codepoint, std::string expected) { char buffer[4096]; MemoryPool pool(buffer, 4096); - StringCopier str(pool); + StringCopier str(&pool); str.startString(); CAPTURE(codepoint); diff --git a/src/ArduinoJson/Deserialization/deserialize.hpp b/src/ArduinoJson/Deserialization/deserialize.hpp index 805cbb04c..e4d27be34 100644 --- a/src/ArduinoJson/Deserialization/deserialize.hpp +++ b/src/ArduinoJson/Deserialization/deserialize.hpp @@ -14,9 +14,10 @@ namespace ARDUINOJSON_NAMESPACE { template