Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow to continue after parse error #4522

Draft
wants to merge 3 commits into
base: develop
Choose a base branch
from
Draft

Allow to continue after parse error #4522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
17b5262
:recycle: allow to continue after parse error
nlohmann Nov 29, 2024
7c97655
:recycle: allow to continue after parse error
nlohmann Nov 30, 2024
cd1e38d
:recycle: allow to continue after parse error
nlohmann Nov 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 46 additions & 22 deletions include/nlohmann/detail/input/parser.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,12 @@ class parser
// parse key
if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr)))
{
return false;
}
}
if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
{
Expand All @@ -224,9 +227,12 @@ class parser
// parse separator (:)
if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr)))
{
return false;
}
}

// remember we are now inside an object
Expand Down Expand Up @@ -267,9 +273,12 @@ class parser

if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res)))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr)))
{
return false;
}
}

if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string())))
Expand Down Expand Up @@ -337,24 +346,33 @@ class parser
case token_type::parse_error:
{
// using "uninitialized" to avoid "expected" message
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr)))
{
return false;
}
break;
}

case token_type::end_of_input:
{
if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(),
"attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(),
"attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr)))
{
return false;
}
}

return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));
}

case token_type::uninitialized:
case token_type::end_array:
case token_type::end_object:
Expand Down Expand Up @@ -422,9 +440,12 @@ class parser
// parse key
if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::value_string))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr)))
{
return false;
}
}

if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
Expand All @@ -435,9 +456,12 @@ class parser
// parse separator (:)
if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr)))
{
return false;
}
}

// parse values
Expand Down
68 changes: 46 additions & 22 deletions single_include/nlohmann/json.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12568,9 +12568,12 @@ class parser
// parse key
if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr)))
{
return false;
}
}
if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
{
Expand All @@ -12580,9 +12583,12 @@ class parser
// parse separator (:)
if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr)))
{
return false;
}
}

// remember we are now inside an object
Expand Down Expand Up @@ -12623,9 +12629,12 @@ class parser

if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res)))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr)))
{
return false;
}
}

if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string())))
Expand Down Expand Up @@ -12693,24 +12702,33 @@ class parser
case token_type::parse_error:
{
// using "uninitialized" to avoid "expected" message
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr)))
{
return false;
}
break;
}

case token_type::end_of_input:
{
if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(),
"attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(),
"attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr)))
{
return false;
}
}

return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));
}

case token_type::uninitialized:
case token_type::end_array:
case token_type::end_object:
Expand Down Expand Up @@ -12778,9 +12796,12 @@ class parser
// parse key
if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::value_string))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr)))
{
return false;
}
}

if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
Expand All @@ -12791,9 +12812,12 @@ class parser
// parse separator (:)
if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
{
return sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
if (!sax->parse_error(m_lexer.get_position(),
m_lexer.get_token_string(),
parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr)))
{
return false;
}
}

// parse values
Expand Down
38 changes: 34 additions & 4 deletions tests/src/unit-class_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,15 +117,28 @@ class SaxEventLogger
return true;
}

bool parse_error(std::size_t position, const std::string& /*unused*/, const json::exception& /*unused*/)
bool parse_error(std::size_t position, const std::string& msg, const json::exception& exception)
{
errored = true;
events.push_back("parse_error(" + std::to_string(position) + ")");
return false;
events.push_back("parse_error(position=" + std::to_string(position) + ", token=<" + msg + ">, exception=" + exception.what() + ")");
return return_value_for_parse_error;
}

std::vector<std::string> events {}; // NOLINT(readability-redundant-member-init)
bool errored = false;
const bool return_value_for_parse_error = false;
std::string event_string()
{
return std::accumulate(events.begin(), events.end(), std::string(),
[](const std::string & a, const std::string & b)
{
return a.empty() ? b : a + '\n' + b;
});
}

explicit SaxEventLogger(bool continue_after_parse_error)
: return_value_for_parse_error(continue_after_parse_error)
{}
};

class SaxCountdown : public nlohmann::json::json_sax_t
Expand Down Expand Up @@ -244,7 +257,7 @@ bool accept_helper(const std::string& s)
CHECK(ok_noexcept == ok_accept);

// 4. parse with SAX (compare with relaxed accept result)
SaxEventLogger el;
SaxEventLogger el(false);
CHECK_NOTHROW(json::sax_parse(s, &el, json::input_format_t::json, false));
CHECK(json::parser(nlohmann::detail::input_adapter(s)).accept(false) == !el.errored);

Expand Down Expand Up @@ -1680,6 +1693,23 @@ TEST_CASE("parser class")
CHECK(json::sax_parse("\"foo\"", &s) == false);
}
}

SECTION("SAX parser continuing after parse error")
{
SaxEventLogger sax(true);

SECTION("Foo")
{
CHECK(json::sax_parse("[{1}, \"a\"]", &sax));
CHECK(sax.event_string() == "start_array()\n"
"start_object()\n"
"parse_error(position=3, token=<1>, exception=[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing object key - unexpected number literal; expected string literal)\n"
"key(1)\n"
"parse_error(position=4, token=<1}>, exception=[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing object separator - unexpected '}'; expected ':')\n"
"parse_error(position=5, token=<1},>, exception=[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - unexpected ','; expected '[', '{', or a literal)\n"
"parse_error(position=9, token=<\"a\">, exception=[json.exception.parse_error.101] parse error at line 1, column 9: syntax error while parsing value - unexpected string literal; expected end of input)");
}
}
}

SECTION("error messages for comments")
Expand Down
Loading