/******************************************************************************** * Copyright (c) 2025-2026 ZF Friedrichshafen AG * * This program and the accompanying materials are made available under the * terms of the Apache License Version 2.0 which is available at * https://www.apache.org/licenses/LICENSE-2.0 * * SPDX-License-Identifier: Apache-2.0 * * Contributors: * Martin Stimpfl - initial API and implementation * Erik Verhoeven - writing TOML and whitespace preservation ********************************************************************************/ #include #include #include #include "../../../sdv_services/core/toml_parser/lexer_toml.h" #include "../../../sdv_services/core/toml_parser/exception.h" /* Requirements Lexer * - Lexer has to ignore all whitespaces outside of comments, basic strings and string literals * - The Lexer has to recognize line comments * - Control characters other than tab (U+0000 to U+0008, U+000A to U+001F, U+007F) are not permitted in comments * - The Lexer has to recognize key-value pairs * - with value data types * - token_quoted_string * - token_integer * - token_float * - token_boolean * - Offset Date-Time (not included) * - Local Date-Time (not included) * - Local Date (not included) * - Local Time (not included) * - Array * - Inline Table * - Keys are on the left of the equals sign and values are on the right * - Keys are always interpreted as strings * - The key, equals sign, and value must be on the same line (though some values can be broken over multiple lines) * - There must be a newline (or EOF) after a key-value pair. (See Inline Table for exceptions.) * - keys * - bare keys may contain only 'A-Za-z0-9_-'; must at least be 1 character long * - quoted keys may be basic strings or string literals; may be empty * - dotted keys are bare keys and dotted keys joined by a dot '.' * - Strings * - may only contain valid UTF-8 characters * - Basic token_quoted_string * - surrounded by '"' * - chracters that must be escaped: '"', '\', and the control characters other than tab (U+0000 to U+0008, U+000A to U+001F, U+007F) * \b - backspace (U+0008) \t - tab (U+0009) \n - linefeed (U+000A) \f - form feed (U+000C) \r - carriage return (U+000D) \" - quote (U+0022) \\ - backslash (U+005C) \uXXXX - unicode (U+XXXX) \UXXXXXXXX - unicode (U+XXXXXXXX) Any Unicode character may be escaped with the \uXXXX or \UXXXXXXXX forms. The escape codes must be valid Unicode scalar values. All other escape sequences not listed above are reserved; if they are used, TOML should produce an error. * - Multiline Basic token_quoted_string - surrounded by three quotation marks '"""' on each side - A newline immediately following the opening delimiter will be trimmed - All other whitespace and newline characters remain intact - When the last non-whitespace character on a line is an unescaped \, it will be trimmed along with all whitespace (including newlines) up to the next non-whitespace character or closing delimiter * - Literal * - Mulitline Literal */ /* * Requirements Lexer * - Tokenize input * - find identifiers (keys; bare and quoted) * + Keys * - find syntax tokens ([, ], =, ., {, }, newline, [[, ]]) outside of literals * + NewLine * + Brackets [] * + Assignment = * + Dot . * + Braces {} * + TableArray [[]] * - find and convert/unescape literals (strings, integers, floats, bools, dates, times) * + token_integer * + token_float * + token_boolean * + Strings * - Offset Date-Time * - Local Date-Time * - Local Date * - Local Time * - ignore whitespaces outside strings * - ignore comments * - invalid input results in an token_error-Token * + invalid bare key * + invalid quoted key * + invalid token_quoted_string * + invalid token_integer * + invalid token_float * + invalid token_boolean * - invalid Offset Date-Time * - invalid Local Date-Time * - invalid Local Date * - invalid Local Time * ? Line and Position of tokens in source file * + Peek() returns the next token without advancing the read location * + Peek(n) returns the next n-th token without advancing the read location * + Consume() returns the next token and advances the read location directly after the returned token * + Consume(n) returns the next n-th token and advances the read location directly after the returned token * + when Peek or Consume would access a token after EOF, they return an empty token, * if lexing was terminated, they return a TerminatedToken * + Exceptions thrown by the CharacterReaderare caught and result in a token_terminated-Token and no further lexing will be done */ TEST(TOMLLexerTest, Keys) { using namespace std::string_literals; // U+1F92B is the Finger-On-Lips Shushing emoji with UTF-8 byte representation 0xF09FA4AB toml_parser::CLexer lexer(R"( key1 = "value1" "key\u0032" = "value2" "key3\U0001F92B" = "abc" '' = "valid" 'literal "key"' = "also valid" "quoted 'key'" = "also valid" _0xabcd = 0xabcd 1234 = 1234 )"s); // Tokens: // NewLine // key1, '=', value1, NewLine // key2, '=', value2, NewLine // key30xF09FA4AB, '=', abc, NewLine // , '=', valid, NewLine // literal "key", '=', also valid const toml_parser::CToken& key1 = lexer.Consume(1); const toml_parser::CToken& key2 = lexer.Consume(3); const toml_parser::CToken& key3 = lexer.Consume(3); const toml_parser::CToken& key4 = lexer.Consume(3); const toml_parser::CToken& key5 = lexer.Consume(3); const toml_parser::CToken& key6 = lexer.Consume(3); const toml_parser::CToken& key7 = lexer.Consume(3); const toml_parser::CToken& key8 = lexer.Consume(3); ASSERT_TRUE(key1); ASSERT_TRUE(key2); ASSERT_TRUE(key3); ASSERT_TRUE(key4); ASSERT_TRUE(key5); ASSERT_TRUE(key6); ASSERT_TRUE(key7); ASSERT_TRUE(key8); EXPECT_EQ(toml_parser::ETokenCategory::token_key, key1.Category()); EXPECT_EQ("key1", key1.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, key2.Category()); EXPECT_EQ("key\u0032", key2.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, key3.Category()); EXPECT_EQ(u8"key3\U0001F92B",key3.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, key4.Category()); EXPECT_EQ("", key4.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, key5.Category()); EXPECT_EQ("literal \"key\"", key5.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, key6.Category()); EXPECT_EQ("quoted 'key'", key6.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, key7.Category()); EXPECT_EQ("_0xabcd", key7.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, key8.Category()); EXPECT_EQ("1234", key8.StringValue()); } TEST(TOMLLexerTest, SyntaxToken_NewLine) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( key1 = "value" )"s); // Tokens: // NewLine // key1, '=', value, NewLine const toml_parser::CToken& newLine1 = lexer.Peek(0); const toml_parser::CToken& newLine2 = lexer.Peek(4); ASSERT_TRUE(newLine1); ASSERT_TRUE(newLine2); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, newLine1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, newLine2.Category()); } TEST(TOMLLexerTest, SyntaxToken_Bracket) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( [table] array = [[1,2],[3,4]] emptyArray = [] [["table array"]] )"s); // Tokens: // NewLine // '[', table, ']', NewLine // array, '=', '[', '[', 1, ',', 2, ']', ',', '[', 3, ',', 4, ']', ']', NewLine // emptyArray, '=', '[', ']', NewLine // '[[', "table array", ']]', NewLine const toml_parser::CToken& tableBracketOpen = lexer.Peek(1); const toml_parser::CToken& tableBracketClose = lexer.Peek(3); const toml_parser::CToken& arrayOfArraysBracketOpen = lexer.Peek(7); const toml_parser::CToken& array1BracketOpen = lexer.Peek(8); const toml_parser::CToken& array1BracketClose = lexer.Peek(12); const toml_parser::CToken& array2BracketOpen = lexer.Peek(14); const toml_parser::CToken& array2BracketClose = lexer.Peek(18); const toml_parser::CToken& arrayOfArraysBracketClose = lexer.Peek(19); const toml_parser::CToken& emptyArrayOpen = lexer.Peek(23); const toml_parser::CToken& emptyArrayClose = lexer.Peek(24); const toml_parser::CToken& tableArrayBracketOpen = lexer.Peek(26); const toml_parser::CToken& tableArrayBracketClose = lexer.Peek(28); ASSERT_TRUE(tableBracketOpen); ASSERT_TRUE(tableBracketClose); ASSERT_TRUE(arrayOfArraysBracketOpen); ASSERT_TRUE(array1BracketOpen); ASSERT_TRUE(array1BracketClose); ASSERT_TRUE(array2BracketOpen); ASSERT_TRUE(array2BracketClose); ASSERT_TRUE(arrayOfArraysBracketClose); ASSERT_TRUE(emptyArrayOpen); ASSERT_TRUE(emptyArrayClose); ASSERT_TRUE(tableArrayBracketOpen); ASSERT_TRUE(tableArrayBracketClose); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_open, tableBracketOpen.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_close, tableBracketClose.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_open, arrayOfArraysBracketOpen.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_open, array1BracketOpen.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_close, array1BracketClose.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_open, array2BracketOpen.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_close, array2BracketClose.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_close, arrayOfArraysBracketClose.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayBracketOpen.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayBracketClose.Category()); } TEST(TOMLLexerTest, SyntaxToken_Assignment) { using namespace std::string_literals; toml_parser::CLexer lexerNormal(R"( key = "value" )"s); // Tokens: // NewLine // key, '=', value, NewLine const toml_parser::CToken& rKeyNormal = lexerNormal.Consume(1); const toml_parser::CToken& rAssignmentNormal = lexerNormal.Consume(); const toml_parser::CToken& rValueNormal = lexerNormal.Consume(); EXPECT_TRUE(rKeyNormal); EXPECT_TRUE(rAssignmentNormal); EXPECT_TRUE(rValueNormal); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNormal.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNormal.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, rValueNormal.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, rValueNormal.StringType()); EXPECT_EQ(rKeyNormal.StringValue(), "key"); EXPECT_EQ(rValueNormal.StringValue(), "value"); toml_parser::CLexer lexerNoSpace(R"(key="value")"s); // Tokens: // key, '=', value const toml_parser::CToken& rKeyNoSpace = lexerNoSpace.Consume(); const toml_parser::CToken& rAssignmentNoSpace = lexerNoSpace.Consume(); const toml_parser::CToken& rValueNoSpace = lexerNoSpace.Consume(); EXPECT_TRUE(rKeyNoSpace); EXPECT_TRUE(rAssignmentNoSpace); EXPECT_TRUE(rValueNoSpace); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpace.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpace.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, rValueNoSpace.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, rValueNoSpace.StringType()); EXPECT_EQ(rKeyNoSpace.StringValue(), "key"); EXPECT_EQ(rValueNoSpace.StringValue(), "value"); } TEST(TOMLLexerTest, Comments) { using namespace std::string_literals; toml_parser::CLexer lexerNormal(R"( #pre key = "value" #follow #next )"s); // Tokens: // NewLine // key, '=', value, NewLine const toml_parser::CToken& rKeyNormal = lexerNormal.Consume(2); const toml_parser::CToken& rAssignmentNormal = lexerNormal.Consume(); const toml_parser::CToken& rValueNormal = lexerNormal.Consume(); EXPECT_TRUE(rKeyNormal); EXPECT_TRUE(rAssignmentNormal); EXPECT_TRUE(rValueNormal); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNormal.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNormal.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, rValueNormal.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, rValueNormal.StringType()); EXPECT_EQ(rKeyNormal.StringValue(), "key"); EXPECT_EQ(rValueNormal.StringValue(), "value"); toml_parser::CLexer lexerNoSpaceString(R"(key="value"#follow)"s); // Tokens: // key, '=', value const toml_parser::CToken& rKeyNoSpaceString = lexerNoSpaceString.Consume(); const toml_parser::CToken& rAssignmentNoSpaceString = lexerNoSpaceString.Consume(); const toml_parser::CToken& rValueNoSpaceString = lexerNoSpaceString.Consume(); EXPECT_TRUE(rKeyNoSpaceString); EXPECT_TRUE(rAssignmentNoSpaceString); EXPECT_TRUE(rValueNoSpaceString); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceString.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceString.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, rValueNoSpaceString.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, rValueNoSpaceString.StringType()); EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceString.Next().Category()); EXPECT_EQ(rKeyNoSpaceString.StringValue(), "key"); EXPECT_EQ(rValueNoSpaceString.StringValue(), "value"); toml_parser::CLexer lexerNoSpaceInteger(R"(key=1234#follow)"s); // Tokens: // key, '=', value const toml_parser::CToken& rKeyNoSpaceInteger = lexerNoSpaceInteger.Consume(); const toml_parser::CToken& rAssignmentNoSpaceInteger = lexerNoSpaceInteger.Consume(); const toml_parser::CToken& rValueNoSpaceInteger = lexerNoSpaceInteger.Consume(); EXPECT_TRUE(rKeyNoSpaceInteger); EXPECT_TRUE(rAssignmentNoSpaceInteger); EXPECT_TRUE(rValueNoSpaceInteger); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceInteger.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceInteger.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, rValueNoSpaceInteger.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceInteger.Next().Category()); EXPECT_EQ(rKeyNoSpaceInteger.StringValue(), "key"); EXPECT_EQ(rValueNoSpaceInteger.IntegerValue(), 1234); toml_parser::CLexer lexerNoSpaceBoolean(R"(key=true#follow)"s); // Tokens: // key, '=', value const toml_parser::CToken& rKeyNoSpaceBoolean = lexerNoSpaceBoolean.Consume(); const toml_parser::CToken& rAssignmentNoSpaceBoolean = lexerNoSpaceBoolean.Consume(); const toml_parser::CToken& rValueNoSpaceBoolean = lexerNoSpaceBoolean.Consume(); EXPECT_TRUE(rKeyNoSpaceBoolean); EXPECT_TRUE(rAssignmentNoSpaceBoolean); EXPECT_TRUE(rValueNoSpaceBoolean); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceBoolean.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceBoolean.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_boolean, rValueNoSpaceBoolean.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceBoolean.Next().Category()); EXPECT_EQ(rKeyNoSpaceBoolean.StringValue(), "key"); EXPECT_EQ(rValueNoSpaceBoolean.BooleanValue(), true); toml_parser::CLexer lexerNoSpaceFloat(R"(key=123.456e78#follow)"s); // Tokens: // key, '=', value const toml_parser::CToken& rKeyNoSpaceFloat = lexerNoSpaceFloat.Consume(); const toml_parser::CToken& rAssignmentNoSpaceFloat = lexerNoSpaceFloat.Consume(); const toml_parser::CToken& rValueNoSpaceFloat = lexerNoSpaceFloat.Consume(); EXPECT_TRUE(rKeyNoSpaceFloat); EXPECT_TRUE(rAssignmentNoSpaceFloat); EXPECT_TRUE(rValueNoSpaceFloat); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceFloat.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceFloat.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, rValueNoSpaceFloat.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceFloat.Next().Category()); EXPECT_EQ(rKeyNoSpaceFloat.StringValue(), "key"); EXPECT_EQ(rValueNoSpaceFloat.FloatValue(), 123.456e78); toml_parser::CLexer lexerNoSpaceInlineTable(R"(key={x=1,y=2,z=3}#follow)"s); // Tokens: // key, '=', value const toml_parser::CToken& rKeyNoSpaceInlineTable = lexerNoSpaceInlineTable.Consume(); const toml_parser::CToken& rAssignmentNoSpaceInlineTable = lexerNoSpaceInlineTable.Consume(); const toml_parser::CToken& rValueNoSpaceInlineTable = lexerNoSpaceInlineTable.Consume(); EXPECT_TRUE(rKeyNoSpaceInlineTable); EXPECT_TRUE(rAssignmentNoSpaceInlineTable); EXPECT_TRUE(rValueNoSpaceInlineTable); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceInlineTable.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceInlineTable.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, rValueNoSpaceInlineTable.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceInlineTable.Next(12).Category()); EXPECT_EQ(rKeyNoSpaceInlineTable.StringValue(), "key"); toml_parser::CLexer lexerNoSpaceArray(R"(key=[1,2,3]#follow)"s); // Tokens: // key, '=', value const toml_parser::CToken& rKeyNoSpaceArray = lexerNoSpaceArray.Consume(); const toml_parser::CToken& rAssignmentNoSpaceArray = lexerNoSpaceArray.Consume(); const toml_parser::CToken& rValueNoSpaceArray = lexerNoSpaceArray.Consume(); EXPECT_TRUE(rKeyNoSpaceArray); EXPECT_TRUE(rAssignmentNoSpaceArray); EXPECT_TRUE(rValueNoSpaceArray); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceArray.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceArray.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_open, rValueNoSpaceArray.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceArray.Next(6).Category()); EXPECT_EQ(rKeyNoSpaceArray.StringValue(), "key"); toml_parser::CLexer lexerNoSpaceTable(R"([key]#follow)"s); // Tokens: // key, '=', value const toml_parser::CToken& rNoSpaceTableOpen = lexerNoSpaceTable.Consume(); EXPECT_TRUE(rNoSpaceTableOpen); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_open, rNoSpaceTableOpen.Category()); const toml_parser::CToken& rKeyNoSpaceTable = lexerNoSpaceTable.Consume(); EXPECT_TRUE(rKeyNoSpaceTable); EXPECT_EQ(rKeyNoSpaceTable.StringValue(), "key"); EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceTable.Category()); const toml_parser::CToken& rNoSpaceTableClose = lexerNoSpaceTable.Consume(); EXPECT_TRUE(rNoSpaceTableClose); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_close, rNoSpaceTableClose.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rNoSpaceTableClose.Next().Category()); } TEST(TOMLLexerTest, SyntaxToken_Dot) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( fruit.name = "banana" # this is best practice fruit. color = "yellow" # same as fruit.color fruit . flavor = "banana" # same as fruit.flavor a.b.c = 2 [dog."tater.man"] )"s); // Tokens: // NewLine // fruit, '.', name, '=', banana, NewLine // fruit, '.', color, '=', yellow, NewLine // fruit, '.', flavor, '=', banana, NewLine // a, '.', b, '.', c, '=', 2, NewLine // '[', dog, '.', tater.man, ']', NewLine const toml_parser::CToken& dot1 = lexer.Peek(2); const toml_parser::CToken& dot2 = lexer.Peek(8); const toml_parser::CToken& dot3 = lexer.Peek(14); const toml_parser::CToken& dot4 = lexer.Peek(20); const toml_parser::CToken& dot5 = lexer.Peek(22); const toml_parser::CToken& dot6 = lexer.Peek(29); ASSERT_TRUE(dot1); ASSERT_TRUE(dot2); ASSERT_TRUE(dot3); ASSERT_TRUE(dot4); ASSERT_TRUE(dot5); ASSERT_TRUE(dot6); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot4.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot5.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot6.Category()); } TEST(TOMLLexerTest, SyntaxToken_Braces) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( inlineTable = { name = "Some Name", type = "Some Type" } nestedInlineTable = { a = { a = 0, b = "b"}, e = { c = 0, d = "d"} } emptyTable = {} )"s); // Tokens: // NewLine // inlineTable, '=', '{', name, '=', Some Name, ',', type, '=', Some Type, '}', NewLine // nestedInlineTable, '=', '{', a, '=', '{', a, '=', 0, ',' b '=', b, '}', ',' b, '=', '{', c, '=', 0, ',' d, '=', d, '}' '}', // NewLine emptyTable, '=', '{', '}', NewLine const toml_parser::CToken& braceOpen = lexer.Peek(3); const toml_parser::CToken& braceClose = lexer.Peek(11); const toml_parser::CToken& nestedOpen1 = lexer.Peek(15); const toml_parser::CToken& nestedOpen2 = lexer.Peek(18); const toml_parser::CToken& nestedClose2 = lexer.Peek(26); const toml_parser::CToken& nestedOpen3 = lexer.Peek(30); const toml_parser::CToken& nestedClose3 = lexer.Peek(38); const toml_parser::CToken& nestedClose1 = lexer.Peek(39); const toml_parser::CToken& emptyTableOpen = lexer.Peek(43); const toml_parser::CToken& emptyTableClose = lexer.Peek(44); ASSERT_TRUE(braceOpen); ASSERT_TRUE(braceClose); ASSERT_TRUE(nestedOpen1); ASSERT_TRUE(nestedOpen2); ASSERT_TRUE(nestedClose2); ASSERT_TRUE(nestedOpen3); ASSERT_TRUE(nestedClose3); ASSERT_TRUE(nestedClose1); ASSERT_TRUE(emptyTableOpen); ASSERT_TRUE(emptyTableClose); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, braceOpen.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, braceClose.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, nestedOpen1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, nestedClose1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, nestedOpen2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, nestedClose2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, nestedOpen3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, nestedClose3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, emptyTableOpen.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, emptyTableClose.Category()); const toml_parser::CToken& a = lexer.Peek(16); const toml_parser::CToken& a_a = lexer.Peek(19); const toml_parser::CToken& a_b = lexer.Peek(23); const toml_parser::CToken& e = lexer.Peek(28); const toml_parser::CToken& e_c = lexer.Peek(31); const toml_parser::CToken& e_d = lexer.Peek(35); const toml_parser::CToken& emptyTable = lexer.Peek(41); EXPECT_EQ(toml_parser::ETokenCategory::token_key, a.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, a_a.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, a_b.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, e.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, e_c.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, e_d.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, emptyTable.Category()); } TEST(TOMLLexerTest, SyntaxToken_ArrayTable) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( [[tableArray]] [[fruits.varieties]] [["this is also a 'key'"]] [['this is also a "key"']] [[a."b.c"]] )"s); // Tokens: // NewLine // '[[', tableArray, ']]', NewLine // '[[', fruits, '.', varieties, ']]', Newline // '[[', "this is also a 'key'", ']]', NewLine // '[[', 'this is also a "key"', ']]', NewLine // '[[', a, '.', "b.c", ']]', Newline const toml_parser::CToken& tableArrayOpen1 = lexer.Peek(1); const toml_parser::CToken& tableArrayName1 = lexer.Peek(2); const toml_parser::CToken& tableArrayClose1 = lexer.Peek(3); const toml_parser::CToken& tableArrayOpen2 = lexer.Peek(5); const toml_parser::CToken& tableArrayName2a = lexer.Peek(6); const toml_parser::CToken& tableArrayName2b = lexer.Peek(8); const toml_parser::CToken& tableArrayClose2 = lexer.Peek(9); const toml_parser::CToken& tableArrayOpen3 = lexer.Peek(11); const toml_parser::CToken& tableArrayName3 = lexer.Peek(12); const toml_parser::CToken& tableArrayClose3 = lexer.Peek(13); const toml_parser::CToken& tableArrayOpen4 = lexer.Peek(15); const toml_parser::CToken& tableArrayName4 = lexer.Peek(16); const toml_parser::CToken& tableArrayClose4 = lexer.Peek(17); const toml_parser::CToken& tableArrayOpen5 = lexer.Peek(19); const toml_parser::CToken& tableArrayName5a = lexer.Peek(20); const toml_parser::CToken& tableArrayName5b = lexer.Peek(22); const toml_parser::CToken& tableArrayClose5 = lexer.Peek(23); ASSERT_TRUE(tableArrayOpen1); ASSERT_TRUE(tableArrayName1); ASSERT_TRUE(tableArrayClose1); ASSERT_TRUE(tableArrayOpen2); ASSERT_TRUE(tableArrayName2a); ASSERT_TRUE(tableArrayName2b); ASSERT_TRUE(tableArrayClose2); ASSERT_TRUE(tableArrayOpen3); ASSERT_TRUE(tableArrayName3); ASSERT_TRUE(tableArrayClose3); ASSERT_TRUE(tableArrayOpen4); ASSERT_TRUE(tableArrayName4); ASSERT_TRUE(tableArrayClose4); ASSERT_TRUE(tableArrayOpen5); ASSERT_TRUE(tableArrayName5a); ASSERT_TRUE(tableArrayName5b); ASSERT_TRUE(tableArrayClose5); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName1.Category()); EXPECT_EQ(tableArrayName1.StringValue(), "tableArray"); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName2a.Category()); EXPECT_EQ(tableArrayName2a.StringValue(), "fruits"); EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName2b.Category()); EXPECT_EQ(tableArrayName2b.StringValue(), "varieties"); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName3.Category()); EXPECT_EQ(tableArrayName3.StringValue(), "this is also a 'key'"); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen4.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName4.Category()); EXPECT_EQ(tableArrayName4.StringValue(), "this is also a \"key\""); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose4.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen5.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName5a.Category()); EXPECT_EQ(tableArrayName5a.StringValue(), "a"); EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName5b.Category()); EXPECT_EQ(tableArrayName5b.StringValue(), "b.c"); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose5.Category()); } TEST(TOMLLexerTest, Datatype_Integer) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( int1 = 1 int2 = +13 int3 = -37 int4 = 1_000 int5 = 5_349_221 int6 = 53_49_221 int7 = 1_2_3_4_5 int8 = 0 int9 = +0 int10 = -0 hex1 = 0xDEADBEEF hex2 = 0xdeadbeef hex3 = 0xdead_beef oct1 = 0o01234567 oct2 = 0o755 bin1 = 0b11111111 bin2 = 0b00000001 bin3 = 0b01010101 intErr1 = _1234 intErr2 = 1__234 intErr3 = 1234_ intErr4 = +0xabcd intErr5 = -0xabcd intErr6 = 0xabc__d intErr7 = 0x_abcd intErr8 = 0xabcd_ intErr9 = 0_xabcd intErr10 = 0XDEADBEEF )"s); // Tokens: // NewLine // int1, '=', 1, Newline // int2, '=', 13, NewLine // int3, '=', -37, NewLine // int4, '=', 1000, NewLine // int5, '=', 5349221, NewLine // int6, '=', 5349221, NewLine // int7, '=', 12345, NewLine // int8, '=', 0, NewLine // int9, '=', 0, NewLine // int10, '=', 0, NewLine // hex1, '=', 0xDEADBEEF, NewLine // hex2, '=', 0xDEADBEEF, NewLine // hex3, '=', 0xDEADBEEF, NewLine // oct1, '=', 01234567, NewLine // oct2, '=', 0755, NewLine // bin1, '=', 0xFF, NewLine // bin2, '=', 0x01, NewLine // bin3, '=', 0x55, NewLine // intErr1, '=', _1234, NewLine // intErr2, '=', 1__234, NewLine // intErr3, '=', 1234_, NewLine // intErr4, '=', +0xabcd, NewLine // intErr5, '=', -0xabcd, NewLine // intErr6, '=', 0xabc__d, NewLine // intErr7, '=', 0x_abcd, NewLine // intErr8, '=', 0xabcd_, NewLine // intErr9, '=', 0_xabcd, NewLine // intErr10, '=', 0XDEADBEEF, NewLine const toml_parser::CToken& int1 = lexer.Consume(3); const toml_parser::CToken& int2 = lexer.Consume(3); const toml_parser::CToken& int3 = lexer.Consume(3); const toml_parser::CToken& int4 = lexer.Consume(3); const toml_parser::CToken& int5 = lexer.Consume(3); const toml_parser::CToken& int6 = lexer.Consume(3); const toml_parser::CToken& int7 = lexer.Consume(3); const toml_parser::CToken& int8 = lexer.Consume(3); const toml_parser::CToken& int9 = lexer.Consume(3); const toml_parser::CToken& int10 = lexer.Consume(3); const toml_parser::CToken& hex1 = lexer.Consume(3); const toml_parser::CToken& hex2 = lexer.Consume(3); const toml_parser::CToken& hex3 = lexer.Consume(3); const toml_parser::CToken& oct1 = lexer.Consume(3); const toml_parser::CToken& oct2 = lexer.Consume(3); const toml_parser::CToken& bin1 = lexer.Consume(3); const toml_parser::CToken& bin2 = lexer.Consume(3); const toml_parser::CToken& bin3 = lexer.Consume(3); const toml_parser::CToken& intErr1 = lexer.Consume(3); const toml_parser::CToken& intErr2 = lexer.Consume(3); const toml_parser::CToken& intErr3 = lexer.Consume(3); const toml_parser::CToken& intErr4 = lexer.Consume(3); const toml_parser::CToken& intErr5 = lexer.Consume(3); const toml_parser::CToken& intErr6 = lexer.Consume(3); const toml_parser::CToken& intErr7 = lexer.Consume(3); const toml_parser::CToken& intErr8 = lexer.Consume(3); const toml_parser::CToken& intErr9 = lexer.Consume(3); const toml_parser::CToken& intErr10 = lexer.Consume(3); ASSERT_TRUE(int1); ASSERT_TRUE(int2); ASSERT_TRUE(int3); ASSERT_TRUE(int4); ASSERT_TRUE(int5); ASSERT_TRUE(int6); ASSERT_TRUE(int7); ASSERT_TRUE(int8); ASSERT_TRUE(int9); ASSERT_TRUE(int10); ASSERT_TRUE(hex1); ASSERT_TRUE(hex2); ASSERT_TRUE(hex3); ASSERT_TRUE(oct1); ASSERT_TRUE(oct2); ASSERT_TRUE(bin1); ASSERT_TRUE(bin2); ASSERT_TRUE(bin3); ASSERT_TRUE(intErr1); ASSERT_TRUE(intErr2); ASSERT_TRUE(intErr3); ASSERT_TRUE(intErr4); ASSERT_TRUE(intErr5); ASSERT_TRUE(intErr6); ASSERT_TRUE(intErr7); ASSERT_TRUE(intErr8); ASSERT_TRUE(intErr9); ASSERT_TRUE(intErr10); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int1.Category()); EXPECT_EQ(1, int1.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int2.Category()); EXPECT_EQ(13, int2.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int3.Category()); EXPECT_EQ(-37, int3.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int4.Category()); EXPECT_EQ(1000, int4.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int5.Category()); EXPECT_EQ(5349221, int5.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int6.Category()); EXPECT_EQ(5349221, int6.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int7.Category()); EXPECT_EQ(12345, int7.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int8.Category()); EXPECT_EQ(0, int8.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int9.Category()); EXPECT_EQ(0, int9.IntegerValue()); EXPECT_EQ(int8.IntegerValue(), int9.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int10.Category()); EXPECT_EQ(0, int10.IntegerValue()); EXPECT_EQ(int8.IntegerValue(), int10.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, hex1.Category()); EXPECT_EQ(0xDEADBEEF, hex1.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, hex2.Category()); EXPECT_EQ(0xDEADBEEF, hex2.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, hex3.Category()); EXPECT_EQ(0xDEADBEEF, hex3.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, oct1.Category()); EXPECT_EQ(01234567, oct1.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, oct2.Category()); EXPECT_EQ(0755, oct2.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, bin1.Category()); EXPECT_EQ(0xFF, bin1.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, bin2.Category()); EXPECT_EQ(0x01, bin2.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_integer, bin3.Category()); EXPECT_EQ(0x55, bin3.IntegerValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr4.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr5.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr6.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr7.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr8.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr9.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr10.Category()); } TEST(TOMLLexerTest, Datatype_Float) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( float1 = +1.0 float2 = 3.1415 float3 = -0.01 float4 = 5e+22 float5 = 1e06 float6 = -2E-2 float7 = 6.626e-34 float8 = 6.6_2_6e-3_4 float9 = 224_617.445_991_228 float10 = -0.0 float11 = +0.0 specialFloat12 = inf specialFloat13 = +inf specialFloat14 = -inf specialFloat15 = nan specialFloat16 = +nan specialFloat17 = -nan errFloat18 = 224__617.445_991_228 errFloat19 = _224_617.445_991_228 errFloat20 = 224_617_.445_991_228 errFloat21 = 224_617._445_991_228 errFloat22 = 224_617.445_991_228_ errFloat23 = 224_617.445_991__228 errFloat24 = 6.626_e-34 errFloat25 = 6.626e_-34 errFloat26 = 6.626e-_34 errFloat27 = 6.626e-34_ errFloat28 = 6.626_e-34 )"s); // Tokens: // NewLine // float1, '=', 1.0, Newline // float2, '=', 3.1415, Newline // float3, '=', -0.01, Newline // float4, '=', 5e+22, Newline // float5, '=', 1e+06, Newline // float6, '=', -2e-02, Newline // float7, '=', 6.626e-34, Newline // float8, '=', 6.6_2_6e-3_4, Newline // float9, '=', 224617.445991228, Newline // float10, '=', -0.0f, Newline // float11, '=', 0.0f, Newline // specialFloat12, '=', inf, NewLine // specialFloat13, '=', inf, NewLine // specialFloat14, '=', -inf, NewLine // specialFloat15, '=', qnan, NewLine // specialFloat16, '=', qnan, NewLine // specialFloat17, '=', -qnan, NewLine // errFloat18, '=', 224__617.445_991_228, NewLine // errFloat19, '=', _224_617.445_991_228, NewLine // errFloat20, '=', 224_617_.445_991_228, NewLine // errFloat21, '=', 224_617._445_991_228, NewLine // errFloat22, '=', 224_617.445_991_228_, NewLine // errFloat23, '=', 224_617.445_991__228, NewLine // errFloat24, '=', 6.626_e-34, NewLine // errFloat25, '=', 6.626e_-34, NewLine // errFloat26, '=', 6.626e-_34, NewLine // errFloat27, '=', 6.626e-34_, NewLine // errFloat28, '=', 6.626_e-34, NewLine const toml_parser::CToken& float1 = lexer.Consume(3); const toml_parser::CToken& float2 = lexer.Consume(3); const toml_parser::CToken& float3 = lexer.Consume(3); const toml_parser::CToken& float4 = lexer.Consume(3); const toml_parser::CToken& float5 = lexer.Consume(3); const toml_parser::CToken& float6 = lexer.Consume(3); const toml_parser::CToken& float7 = lexer.Consume(3); const toml_parser::CToken& float8 = lexer.Consume(3); const toml_parser::CToken& float9 = lexer.Consume(3); const toml_parser::CToken& float10 = lexer.Consume(3); const toml_parser::CToken& float11 = lexer.Consume(3); const toml_parser::CToken& specialFloat12 = lexer.Consume(3); const toml_parser::CToken& specialFloat13 = lexer.Consume(3); const toml_parser::CToken& specialFloat14 = lexer.Consume(3); const toml_parser::CToken& specialFloat15 = lexer.Consume(3); const toml_parser::CToken& specialFloat16 = lexer.Consume(3); const toml_parser::CToken& specialFloat17 = lexer.Consume(3); const toml_parser::CToken& errFloat18 = lexer.Consume(3); const toml_parser::CToken& errFloat19 = lexer.Consume(3); const toml_parser::CToken& errFloat20 = lexer.Consume(3); const toml_parser::CToken& errFloat21 = lexer.Consume(3); const toml_parser::CToken& errFloat22 = lexer.Consume(3); const toml_parser::CToken& errFloat23 = lexer.Consume(3); const toml_parser::CToken& errFloat24 = lexer.Consume(3); const toml_parser::CToken& errFloat25 = lexer.Consume(3); const toml_parser::CToken& errFloat26 = lexer.Consume(3); const toml_parser::CToken& errFloat27 = lexer.Consume(3); const toml_parser::CToken& errFloat28 = lexer.Consume(3); ASSERT_TRUE(float1); ASSERT_TRUE(float2); ASSERT_TRUE(float3); ASSERT_TRUE(float4); ASSERT_TRUE(float5); ASSERT_TRUE(float6); ASSERT_TRUE(float7); ASSERT_TRUE(float8); ASSERT_TRUE(float9); ASSERT_TRUE(float10); ASSERT_TRUE(float11); ASSERT_TRUE(specialFloat12); ASSERT_TRUE(specialFloat13); ASSERT_TRUE(specialFloat14); ASSERT_TRUE(specialFloat15); ASSERT_TRUE(specialFloat16); ASSERT_TRUE(specialFloat17); ASSERT_TRUE(errFloat18); ASSERT_TRUE(errFloat19); ASSERT_TRUE(errFloat20); ASSERT_TRUE(errFloat21); ASSERT_TRUE(errFloat22); ASSERT_TRUE(errFloat23); ASSERT_TRUE(errFloat24); ASSERT_TRUE(errFloat25); ASSERT_TRUE(errFloat26); ASSERT_TRUE(errFloat27); ASSERT_TRUE(errFloat28); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float1.Category()); EXPECT_EQ(1.0, float1.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float2.Category()); EXPECT_EQ(3.1415, float2.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float3.Category()); EXPECT_EQ(-0.01, float3.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float4.Category()); EXPECT_EQ(5e+22, float4.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float5.Category()); EXPECT_EQ(1e+06, float5.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float6.Category()); EXPECT_EQ(-2e-02, float6.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float7.Category()); EXPECT_EQ(6.626e-34, float7.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float8.Category()); EXPECT_EQ(6.626e-34, float8.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float9.Category()); EXPECT_EQ(224617.445991228, float9.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float10.Category()); EXPECT_EQ(-0.0, float10.FloatValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_float, float11.Category()); EXPECT_EQ(+0.0, float11.FloatValue()); #ifdef __GNUC__ #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif auto fnMakeUInt = [](double d) { union { double d; uint64_t ui; } temp{ d }; return temp.ui; }; EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat12.Category()); EXPECT_EQ(0x7FF0000000000000ull, fnMakeUInt(specialFloat12.FloatValue())); // inf EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat13.Category()); EXPECT_EQ(0x7FF0000000000000ull, fnMakeUInt(specialFloat13.FloatValue())); // +inf EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat14.Category()); EXPECT_EQ(0xFFF0000000000000ull, fnMakeUInt(specialFloat14.FloatValue())); // -inf EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat15.Category()); EXPECT_EQ(0x7FF0000000000000ull, fnMakeUInt(specialFloat15.FloatValue()) & 0xFFF0000000000000ull); // nan EXPECT_NE(0, fnMakeUInt(specialFloat15.FloatValue()) & 0x000FFFFFFFFFFFFFull); EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat16.Category()); EXPECT_EQ(0x7FF0000000000000ull, fnMakeUInt(specialFloat16.FloatValue()) & 0xFFF0000000000000ull); EXPECT_NE(0, fnMakeUInt(specialFloat16.FloatValue()) & 0x000FFFFFFFFFFFFFull); EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat17.Category()); EXPECT_EQ(0xFFF0000000000000ull, fnMakeUInt(specialFloat17.FloatValue()) & 0xFFF0000000000000ull); EXPECT_NE(0, fnMakeUInt(specialFloat17.FloatValue()) & 0x000FFFFFFFFFFFFFull); #ifdef __GNUC__ #pragma GCC diagnostic pop #endif EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat18.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat19.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat20.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat21.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat22.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat23.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat24.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat25.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat26.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat27.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat28.Category()); } TEST(TOMLLexerTest, Datatype_Boolean) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( key1 = true key2 = false )"s); // Tokens: // NewLine // key1, =, 1, NewLine // key2, =, 0, NewLine const toml_parser::CToken& trueToken = lexer.Consume(3); const toml_parser::CToken& falseToken = lexer.Consume(3); ASSERT_TRUE(trueToken); ASSERT_TRUE(falseToken); EXPECT_EQ(toml_parser::ETokenCategory::token_boolean, trueToken.Category()); EXPECT_TRUE(trueToken.BooleanValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_boolean, falseToken.Category()); EXPECT_FALSE(falseToken.BooleanValue()); } TEST(TOMLLexerTest, Datatype_String_BasicString) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( basicString = "this is a string" basicString_esc1 = "Backspace\b" basicString_esc2 = "Tab\t" basicString_esc3 = "Linefeed\n" basicString_esc4 = "Formfeed\f" basicString_esc5 = "carriage return\r" basicString_esc6 = "quote\"" basicString_esc7 = "backslash\\" basicString_esc8 = "unicode hiragana 'no': \u306e" basicString_esc9 = "Musical eighth note: \U0001D160" )"s); // Tokens: // NewLine // basicString, '=', this is a string, NewLine // basicString_esc1, '=', Bachspac, Newline // basicString_esc2, '=', Tab\t, Newline // basicString_esc3, '=', Linefeed\n, Newline // basicString_esc4, '=', Formfeed\f, Newline // basicString_esc5, '=', carriage return\r, Newline // basicString_esc6, '=', quote", Newline // basicString_esc7, '=', backslash\, Newline // basicString_esc8, '=', の, Newline // basicString_esc9, '=', 𝅘𝅥𝅮, Newline const toml_parser::CToken& string = lexer.Consume(3); const toml_parser::CToken& string_esc1 = lexer.Consume(3); const toml_parser::CToken& string_esc2 = lexer.Consume(3); const toml_parser::CToken& string_esc3 = lexer.Consume(3); const toml_parser::CToken& string_esc4 = lexer.Consume(3); const toml_parser::CToken& string_esc5 = lexer.Consume(3); const toml_parser::CToken& string_esc6 = lexer.Consume(3); const toml_parser::CToken& string_esc7 = lexer.Consume(3); const toml_parser::CToken& string_esc8 = lexer.Consume(3); const toml_parser::CToken& string_esc9 = lexer.Consume(3); ASSERT_TRUE(string); ASSERT_TRUE(string_esc1); ASSERT_TRUE(string_esc2); ASSERT_TRUE(string_esc3); ASSERT_TRUE(string_esc4); ASSERT_TRUE(string_esc5); ASSERT_TRUE(string_esc6); ASSERT_TRUE(string_esc7); ASSERT_TRUE(string_esc8); ASSERT_TRUE(string_esc9); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string.StringType()); EXPECT_EQ("this is a string", string.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc1.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc1.StringType()); EXPECT_EQ("Backspace\b", string_esc1.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc2.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc2.StringType()); EXPECT_EQ("Tab\t", string_esc2.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc3.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc3.StringType()); EXPECT_EQ("Linefeed\n", string_esc3.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc4.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc4.StringType()); EXPECT_EQ("Formfeed\f", string_esc4.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc5.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc5.StringType()); EXPECT_EQ("carriage return\r", string_esc5.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc6.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc6.StringType()); EXPECT_EQ("quote\"", string_esc6.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc7.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc7.StringType()); EXPECT_EQ("backslash\\", string_esc7.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc8.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc8.StringType()); EXPECT_EQ("unicode hiragana 'no': の", string_esc8.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc9.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc9.StringType()); EXPECT_EQ("Musical eighth note: 𝅘𝅥𝅮", string_esc9.StringValue()); } TEST(TOMLLexerTest, Datatype_String_BasicStringMultiline) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( basicStringMultiline1 = """ this is a multiline string""" basicStringMultiline2 = """\ this is a nice \ multiline string""" basicStringMultiline3 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" basicStringMultiline4 = """"This," she said, "is just a pointless statement."""" basicStringMultiline_esc1 = """\ Backspace\b \ multiline""" basicStringMultiline_esc2 = """\ Tab\t\ multiline""" basicStringMultiline_esc3 = """\ Linefeed\n\ multiline""" basicStringMultiline_esc4 = """\ Formfeed\f\ multiline""" basicStringMultiline_esc5 = """\ carriage return\r\ multiline""" basicStringMultiline_esc6 = """\ quote\" \ multiline""" basicStringMultiline_esc7 = """\ backslash\\\ multiline""" basicStringMultiline_esc8 = """\ unicode hiragana 'no': \u306e \ multiline""" basicStringMultiline_esc9 = """\ Musical eighth note: \U0001D160 \ multiline""" )"s); // Tokens: // NewLine // basicStringMultiline1, '=', this is a\nmultiline string, NewLine // basicStringMultiline2, '=', this is a nice multiline string, NewLine // basicStringMultiline3, '=', Here are fifteen quotation marks: """""""""""""""., NewLine // basicStringMultiline4, '=', "This," she said, "is just a pointless statement.", NewLine // basicStringMultiline_esc1, '=', Backspac multiline, Newline // basicStringMultiline_esc2, '=', Tab\tmultiline, Newline // basicStringMultiline_esc3, '=', Linefeed\nmultiline, Newline // basicStringMultiline_esc4, '=', Formfeed\fmultiline, Newline // basicStringMultiline_esc5, '=', carriage return\rmultiline, Newline // basicStringMultiline_esc6, '=', quote" multiline, Newline // basicStringMultiline_esc7, '=', backslash\multiline, Newline // basicStringMultiline_esc8, '=', unicode hiragana 'no': の multiline, Newline // basicStringMultiline_esc9, '=', Musical eighth note: 𝅘𝅥𝅮 multiline, Newline const toml_parser::CToken& string1 = lexer.Consume(3); const toml_parser::CToken& string2 = lexer.Consume(3); const toml_parser::CToken& string3 = lexer.Consume(3); const toml_parser::CToken& string4 = lexer.Consume(3); const toml_parser::CToken& string_esc1 = lexer.Consume(3); const toml_parser::CToken& string_esc2 = lexer.Consume(3); const toml_parser::CToken& string_esc3 = lexer.Consume(3); const toml_parser::CToken& string_esc4 = lexer.Consume(3); const toml_parser::CToken& string_esc5 = lexer.Consume(3); const toml_parser::CToken& string_esc6 = lexer.Consume(3); const toml_parser::CToken& string_esc7 = lexer.Consume(3); const toml_parser::CToken& string_esc8 = lexer.Consume(3); const toml_parser::CToken& string_esc9 = lexer.Consume(3); ASSERT_TRUE(string1); ASSERT_TRUE(string2); ASSERT_TRUE(string3); ASSERT_TRUE(string4); ASSERT_TRUE(string_esc1); ASSERT_TRUE(string_esc2); ASSERT_TRUE(string_esc3); ASSERT_TRUE(string_esc4); ASSERT_TRUE(string_esc5); ASSERT_TRUE(string_esc6); ASSERT_TRUE(string_esc7); ASSERT_TRUE(string_esc8); ASSERT_TRUE(string_esc9); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string1.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string1.StringType()); EXPECT_EQ("this is a\nmultiline string", string1.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string2.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string2.StringType()); EXPECT_EQ("this is a nice multiline string", string2.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string3.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string3.StringType()); EXPECT_EQ("Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".", string3.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string4.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string4.StringType()); EXPECT_EQ("\"This,\" she said, \"is just a pointless statement.\"", string4.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc1.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc1.StringType()); EXPECT_EQ("Backspace\b multiline", string_esc1.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc2.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc2.StringType()); EXPECT_EQ("Tab\tmultiline", string_esc2.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc3.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc3.StringType()); EXPECT_EQ("Linefeed\nmultiline", string_esc3.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc4.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc4.StringType()); EXPECT_EQ("Formfeed\fmultiline", string_esc4.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc5.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc5.StringType()); EXPECT_EQ("carriage return\rmultiline", string_esc5.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc6.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc6.StringType()); EXPECT_EQ("quote\" multiline", string_esc6.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc7.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc7.StringType()); EXPECT_EQ("backslash\\multiline", string_esc7.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc8.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc8.StringType()); EXPECT_EQ("unicode hiragana 'no': の multiline", string_esc8.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc9.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc9.StringType()); EXPECT_EQ("Musical eighth note: 𝅘𝅥𝅮 multiline", string_esc9.StringValue()); } TEST(TOMLLexerTest, Datatype_String_LiteralString) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( winpath = 'C:\Users\nodejs\templates' winpath2 = '\\ServerX\admin$\system32\' quoted = 'Tom "Dubs" Preston-Werner' regex = '<\i\c*\s*>' )"s); // Tokens: // NewLine // winpath, '=', C:\Users\nodejs\templates, NewLine // winpath2, '=', \\ServerX\admin$\system32\, NewLine // quoted, '=', Tom "Dubs" Preston-Werner, NewLine // regex, '=', <\i\c*\s*>, NewLine const toml_parser::CToken& winpath = lexer.Consume(3); const toml_parser::CToken& winpath2 = lexer.Consume(3); const toml_parser::CToken& quoted = lexer.Consume(3); const toml_parser::CToken& regex = lexer.Consume(3); ASSERT_TRUE(winpath); ASSERT_TRUE(winpath2); ASSERT_TRUE(quoted); ASSERT_TRUE(regex); EXPECT_EQ(toml_parser::ETokenCategory::token_string, winpath.Category()); EXPECT_EQ(toml_parser::ETokenStringType::literal_string, winpath.StringType()); EXPECT_EQ("C:\\Users\\nodejs\\templates", winpath.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, winpath2.Category()); EXPECT_EQ(toml_parser::ETokenStringType::literal_string, winpath2.StringType()); EXPECT_EQ("\\\\ServerX\\admin$\\system32\\", winpath2.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, quoted.Category()); EXPECT_EQ(toml_parser::ETokenStringType::literal_string, quoted.StringType()); EXPECT_EQ("Tom \"Dubs\" Preston-Werner", quoted.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, regex.Category()); EXPECT_EQ(toml_parser::ETokenStringType::literal_string, regex.StringType()); EXPECT_EQ("<\\i\\c*\\s*>", regex.StringValue()); } TEST(TOMLLexerTest, Datatype_String_LiteralStringMultiline) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( regex2 = '''I [dw]on't need \d{2} apples''' lines = ''' The first newline is trimmed in raw strings. All other whitespace is preserved. ''' quot15 = '''Here are fifteen quotation marks: """""""""""""""''' apos15 = "Here are fifteen apostrophes: '''''''''''''''" str = ''''That,' she said, 'is still pointless.'''' )"s); // Tokens: // NewLine // regex2, '=', I't need \d{2} apples, NewLine // lines, '=', The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n, NewLine // quot15, '=', Here are fifteen quotation marks: """"""""""""""", NewLine // apos15, '=', Here are fifteen apostropes: ''''''''''''''', NewLine // str, '=', 'That,' she said, 'is still pointless.', NewLine const toml_parser::CToken& regex2 = lexer.Consume(3); const toml_parser::CToken& lines = lexer.Consume(3); const toml_parser::CToken& quot15 = lexer.Consume(3); const toml_parser::CToken& apos15 = lexer.Consume(3); const toml_parser::CToken& str = lexer.Consume(3); ASSERT_TRUE(regex2); ASSERT_TRUE(lines); ASSERT_TRUE(quot15); ASSERT_TRUE(apos15); ASSERT_TRUE(str); EXPECT_EQ(toml_parser::ETokenCategory::token_string, regex2.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_literal, regex2.StringType()); EXPECT_EQ("I [dw]on't need \\d{2} apples", regex2.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, lines.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_literal, lines.StringType()); EXPECT_EQ("The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", lines.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, quot15.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_literal, quot15.StringType()); EXPECT_EQ("Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\"", quot15.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, apos15.Category()); EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, apos15.StringType()); EXPECT_EQ("Here are fifteen apostrophes: '''''''''''''''", apos15.StringValue()); EXPECT_EQ(toml_parser::ETokenCategory::token_string, str.Category()); EXPECT_EQ(toml_parser::ETokenStringType::multi_line_literal, str.StringType()); EXPECT_EQ("'That,' she said, 'is still pointless.'", str.StringValue()); } // TEST(TOMLLexerTest, Datatype_OffsetDateTime) // { // ASSERT_TRUE(false); // } // TEST(TOMLLexerTest, Datatype_DateTime) // { // ASSERT_TRUE(false); // } // TEST(TOMLLexerTest, Datatype_LocalDate) // { // ASSERT_TRUE(false); // } // TEST(TOMLLexerTest, Datatype_LocalTime) // { // ASSERT_TRUE(false); // } TEST(TOMLLexerTest, Invalid_Key) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( keyä = "value1" "key\u123" = "value2" "key\U1234" = "value3" "key\k" = "value4" )"s); // Tokens: // NewLine // ErrorToken, '=', value1, NewLine // ErrorToken, '=', value2, NewLine // ErrorToken, '=', value3, NewLine // ErrorToken, '=', value4, NewLine const toml_parser::CToken& invBareKey = lexer.Consume(1); // only [a-zA-Z0-9-_]allowed for bare key const toml_parser::CToken& invQuotedKey1 = lexer.Consume(3); // \u has to be followed by 4 times [0-9A-Fa-F] const toml_parser::CToken& invQuotedKey2 = lexer.Consume(3); // \U has to be followed by 8 times [0-9A-Fa-F] const toml_parser::CToken& invQuotedKey3 = lexer.Consume(3); // any escape sequence except specified in the TOML secs are invalid ASSERT_TRUE(invBareKey); // only [a-zA-Z0-9-_]allowed for bare key ASSERT_TRUE(invQuotedKey1); // \u has to be followed by 4 times [0-9A-Fa-F] ASSERT_TRUE(invQuotedKey2); // \U has to be followed by 8 times [0-9A-Fa-F] ASSERT_TRUE(invQuotedKey3); // any escape sequence except specified in the TOML secs are invalid EXPECT_EQ(toml_parser::ETokenCategory::token_error, invBareKey.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invQuotedKey1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invQuotedKey2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invQuotedKey3.Category()); } TEST(TOMLLexerTest, Invalid_String) { using namespace std::string_literals; toml_parser::CLexer lexer1(R"(key1 = "invalid escape sequence \h)"s); toml_parser::CLexer lexer2(R"(key2 = """invalid escape sequence \h""")"s); toml_parser::CLexer lexer3(R"(key3 = "\u000")"s); toml_parser::CLexer lexer4(R"(key4 = "\U0000000")"s); // Tokens: // key1, '=', ErrorToken // key2, '=', ErrorToken // key3, '=', ErrorToken // key4, '=', ErrorToken const toml_parser::CToken& invString1 = lexer1.Consume(2); const toml_parser::CToken& invString2 = lexer2.Consume(2); const toml_parser::CToken& invString3 = lexer3.Consume(2); const toml_parser::CToken& invString4 = lexer4.Consume(2); ASSERT_TRUE(invString1); ASSERT_TRUE(invString2); ASSERT_TRUE(invString3); ASSERT_TRUE(invString4); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invString1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invString2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invString3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invString4.Category()); } TEST(TOMLLexerTest, Invalid_Integer) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( invInteger3 = 01 invInteger4 = 0a1234 invInteger5 = 1234_ invInteger6 = _1234 invInteger7 = 0x_1234 invInteger8 = 0o_1234 invInteger9 = 0b_1010 invInteger10 = +0x1234 invInteger11 = +0o1234 invInteger12 = +0b1010 invInteger13 = -0x1234 invInteger14 = -0o1234 invInteger15 = -0b1010 )"s); // Tokens: // NewLine // invInteger3, '=', ErrorToken, NewLine // invInteger4, '=', ErrorToken, NewLine // invInteger5, '=', ErrorToken, NewLine // invInteger6, '=', ErrorToken, NewLine // invInteger7, '=', ErrorToken, NewLine // invInteger8, '=', ErrorToken, NewLine // invInteger9, '=', ErrorToken, NewLine // invInteger10, '=', ErrorToken, NewLine // invInteger11, '=', ErrorToken, NewLine // invInteger12, '=', ErrorToken, NewLine // invInteger12, '=', ErrorToken, NewLine // invInteger12, '=', ErrorToken, NewLine // invInteger12, '=', ErrorToken, NewLine const toml_parser::CToken& invInteger3 = lexer.Consume(3); // no leading zeros const toml_parser::CToken& invInteger4 = lexer.Consume(3); // invalid prefix const toml_parser::CToken& invInteger5 = lexer.Consume(3); // underscores only allowed between two digits const toml_parser::CToken& invInteger6 = lexer.Consume(3); // underscores only allowed between two digits const toml_parser::CToken& invInteger7 = lexer.Consume(3); // underscores not allowed after prefix const toml_parser::CToken& invInteger8 = lexer.Consume(3); // underscores not allowed after prefix const toml_parser::CToken& invInteger9 = lexer.Consume(3); // underscores not allowed after prefix const toml_parser::CToken& invInteger10 = lexer.Consume(3); // no leading + allowed const toml_parser::CToken& invInteger11 = lexer.Consume(3); // no leading + allowed const toml_parser::CToken& invInteger12 = lexer.Consume(3); // no leading + allowed const toml_parser::CToken& invInteger13 = lexer.Consume(3); // no negative number representation allowed with hex format const toml_parser::CToken& invInteger14 = lexer.Consume(3); // no negative number representation allowed with ocatal format const toml_parser::CToken& invInteger15 = lexer.Consume(3); // no negative number representation allowed with binary format ASSERT_TRUE(invInteger3); // no leading zeros ASSERT_TRUE(invInteger4); // invalid prefix ASSERT_TRUE(invInteger5); // underscores only allowed between two digits ASSERT_TRUE(invInteger6); // underscores only allowed between two digits ASSERT_TRUE(invInteger7); // underscores not allowed after prefix ASSERT_TRUE(invInteger8); // underscores not allowed after prefix ASSERT_TRUE(invInteger9); // underscores not allowed after prefix ASSERT_TRUE(invInteger10); // no leading + allowed ASSERT_TRUE(invInteger11); // no leading + allowed ASSERT_TRUE(invInteger12); // no leading + allowed ASSERT_TRUE(invInteger13); // no negative number representation allowed with hex format ASSERT_TRUE(invInteger14); // no negative number representation allowed with ocatal format ASSERT_TRUE(invInteger15); // no negative number representation allowed with binary format EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger4.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger5.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger6.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger7.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger8.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger9.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger10.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger11.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger12.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger13.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger14.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger15.Category()); } TEST(TOMLLexerTest, Invalid_Float) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( invFloat1 = .7 invFloat2 = 1. invFloat3 = 3.e4 invFloat4 = Inf invFloat5 = NaN )"s); // Tokens: // NewLine // invFloat1, '=', ErrorToken, NewLine // invFloat2, '=', ErrorToken, NewLine // invFloat3, '=', ErrorToken, NewLine // invFloat4, '=', ErrorToken, NewLine // invFloat5, '=', ErrorToken, NewLine const toml_parser::CToken& invFloat1 = lexer.Consume(3); // decimal point has to be between two digits const toml_parser::CToken& invFloat2 = lexer.Consume(3); // decimal point has to be between two digits const toml_parser::CToken& invFloat3 = lexer.Consume(3); // decimal point has to be between two digits const toml_parser::CToken& invFloat4 = lexer.Consume(3); // special float values are always lower case const toml_parser::CToken& invFloat5 = lexer.Consume(3); // special float values are always lower case ASSERT_TRUE(invFloat1); // decimal point has to be between two digits ASSERT_TRUE(invFloat2); // decimal point has to be between two digits ASSERT_TRUE(invFloat3); // decimal point has to be between two digits ASSERT_TRUE(invFloat4); // special float values are always lower case ASSERT_TRUE(invFloat5); // special float values are always lower case EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat2.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat3.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat4.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat5.Category()); } TEST(TOMLLexerTest, Invalid_Boolean) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( invBool1 = True invBool2 = False )"s); // Tokens: // NewLine // invBool1, '=', ErrorToken, NewLine // invBool2, '=', ErrorToken, NewLine const toml_parser::CToken& invBool1 = lexer.Consume(3); // keyword for true has to be lower case const toml_parser::CToken& invBool2 = lexer.Consume(3); // keyword for false has to be lower case ASSERT_TRUE(invBool1); // keyword for true has to be lower case ASSERT_TRUE(invBool2); // keyword for false has to be lower case EXPECT_EQ(toml_parser::ETokenCategory::token_error, invBool1.Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_error, invBool2.Category()); } // TEST(TOMLLexerTest, Invalid_OffsetDateTime) // { // ASSERT_TRUE(false); // } // TEST(TOMLLexerTest, Invalid_LocalDateTime) // { // ASSERT_TRUE(false); // } // TEST(TOMLLexerTest, Invalid_LocalDate) // { // ASSERT_TRUE(false); // } // TEST(TOMLLexerTest, Invalid_LocalTime) // { // ASSERT_TRUE(false); // } TEST(TOMLLexerTest, Peek_NoAdvance) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( key1 = "value1" key2 = "value2" )"s); // Tokens: // NewLine // key1, '=', value1, NewLine // key2, '=', value2, NewLine ASSERT_TRUE(lexer.Peek()); ASSERT_TRUE(lexer.Peek()); ASSERT_TRUE(lexer.Peek(1)); ASSERT_TRUE(lexer.Peek(1)); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, lexer.Peek().Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, lexer.Peek().Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, lexer.Peek(1).Category()); EXPECT_EQ(toml_parser::ETokenCategory::token_key, lexer.Peek(1).Category()); } TEST(TOMLLexerTest, Consume_Advance) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( key1 = "value1" key2 = "value2" )"s); // Tokens: // NewLine // key1, '=', value1, NewLine // key2, '=', value2, NewLine auto ptr = lexer.Consume(); ASSERT_TRUE(ptr); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, ptr.Category()); ptr = lexer.Consume(); ASSERT_TRUE(ptr); EXPECT_EQ(toml_parser::ETokenCategory::token_key, ptr.Category()); ptr = lexer.Consume(2); ASSERT_TRUE(ptr); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, ptr.Category()); ptr = lexer.Consume(1); ASSERT_TRUE(ptr); EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, ptr.Category()); } TEST(TOMLLexerTest, PeekConsume_BoundsCheck) { using namespace std::string_literals; toml_parser::CLexer lexer(R"( key1 = "value1" key2 = "value2" )"s); // Tokens: // NewLine // key1, '=', value1, NewLine // key2, '=', value2, NewLine EXPECT_FALSE(lexer.Consume(10)); EXPECT_FALSE(lexer.Consume()); EXPECT_FALSE(lexer.Peek()); EXPECT_FALSE(lexer.Peek(2)); EXPECT_FALSE(lexer.Peek(0)); EXPECT_FALSE(lexer.Consume(0)); } TEST(TOMLLexerTest, PeekConsume_EmptyInput) { using namespace std::string_literals; toml_parser::CLexer lexer(R"()"s); // Tokens: EXPECT_FALSE(lexer.Peek()); EXPECT_FALSE(lexer.Peek(1)); EXPECT_FALSE(lexer.Consume()); EXPECT_FALSE(lexer.Consume(1)); } TEST(TOMLLexerTest, ExceptionHandling) { using namespace std::string_literals; toml_parser::CLexer lexer; try { lexer.Feed(std::string("") + static_cast(0xC1)); } catch (const sdv::toml::XTOMLParseException&) { } // Tokens: // token_terminated-Token EXPECT_EQ(toml_parser::ETokenCategory::token_terminated, lexer.Consume().Category()); EXPECT_FALSE(lexer.Consume()); EXPECT_FALSE(lexer.Peek()); EXPECT_FALSE(lexer.Peek(2)); EXPECT_FALSE(lexer.Peek(0)); EXPECT_FALSE(lexer.Consume(0)); } TEST(TOMLLexerTest, DISABLED_RegenerateTOML) { using namespace std::string_literals; std::string ssOrginal = R"( # original comments key1 = "value1" # follow comments "key\u0032" = "value2" "key3\U0001F92B" = "abc" '' = "valid" 'literal "key"' = "also valid" "quoted 'key'" = "also valid" _0xabcd = 0xabcd 1234 = 1234 [table] array = [[1,2],[3,4]] emptyArray = [] [["table array"]] fruit.name = "banana" # this is best practice fruit. color = "yellow" # same as fruit.color fruit . flavor = "banana" # same as fruit.flavor a.b.c = 2 [dog."tater.man"] inlineTable = { name = "Some Name", type = "Some Type" } nestedInlineTable = { a = { a = 0, b = "b"}, e = { c = 0, d = "d"} } emptyTable = {} [[tableArray]] [[fruits.varieties]] [["this is also a 'key'"]] [['this is also a "key"']] [[a."b.c"]] int1 = 1 int2 = +13 int3 = -37 int4 = 1_000 int5 = 5_349_221 int6 = 53_49_221 int7 = 1_2_3_4_5 int8 = 0 int9 = +0 int10 = -0 hex1 = 0xDEADBEEF hex2 = 0xdeadbeef hex3 = 0xdead_beef oct1 = 0o01234567 oct2 = 0o755 bin1 = 0b11111111 bin2 = 0b00000001 bin3 = 0b01010101 intErr1 = _1234 intErr2 = 1__234 intErr3 = 1234_ intErr4 = +0xabcd intErr5 = -0xabcd intErr6 = 0xabc__d intErr7 = 0x_abcd intErr8 = 0xabcd_ intErr9 = 0_xabcd intErr10 = 0XDEADBEEF float1 = +1.0 float2 = 3.1415 float3 = -0.01 float4 = 5e+22 float5 = 1e06 float6 = -2E-2 float7 = 6.626e-34 float8 = 6.6_2_6e-3_4 float9 = 224_617.445_991_228 float10 = -0.0 float11 = +0.0 specialFloat12 = inf specialFloat13 = +inf specialFloat14 = -inf specialFloat15 = nan specialFloat16 = +nan specialFloat17 = -nan errFloat18 = 224__617.445_991_228 errFloat19 = _224_617.445_991_228 errFloat20 = 224_617_.445_991_228 errFloat21 = 224_617._445_991_228 errFloat22 = 224_617.445_991_228_ errFloat23 = 224_617.445_991__228 errFloat24 = 6.626_e-34 errFloat25 = 6.626e_-34 errFloat26 = 6.626e-_34 errFloat27 = 6.626e-34_ errFloat28 = 6.626_e-34 key1 = true key2 = false basicString = "this is a string" basicString_esc1 = "Backspace\b" basicString_esc2 = "Tab\t" basicString_esc3 = "Linefeed\n" basicString_esc4 = "Formfeed\f" basicString_esc5 = "carriage return\r" basicString_esc6 = "quote\"" basicString_esc7 = "backslash\\" basicString_esc8 = "unicode hiragana 'no': \u306e" basicString_esc9 = "Musical eighth note: \U0001D160" basicStringMultiline1 = """ this is a multiline string""" basicStringMultiline2 = """\ this is a nice \ multiline string""" basicStringMultiline3 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" basicStringMultiline4 = """"This," she said, "is just a pointless statement."""" basicStringMultiline_esc1 = """\ Backspace\b \ multiline""" basicStringMultiline_esc2 = """\ Tab\t\ multiline""" basicStringMultiline_esc3 = """\ Linefeed\n\ multiline""" basicStringMultiline_esc4 = """\ Formfeed\f\ multiline""" basicStringMultiline_esc5 = """\ carriage return\r\ multiline""" basicStringMultiline_esc6 = """\ quote\" \ multiline""" basicStringMultiline_esc7 = """\ backslash\\\ multiline""" basicStringMultiline_esc8 = """\ unicode hiragana 'no': \u306e \ multiline""" basicStringMultiline_esc9 = """\ Musical eighth note: \U0001D160 \ multiline""" winpath = 'C:\Users\nodejs\templates' winpath2 = '\\ServerX\admin$\system32\' quoted = 'Tom "Dubs" Preston-Werner' regex = '<\i\c*\s*>' regex2 = '''I [dw]on't need \d{2} apples''' lines = ''' The first newline is trimmed in raw strings. All other whitespace is preserved. ''' quot15 = '''Here are fifteen quotation marks: """""""""""""""''' apos15 = "Here are fifteen apostrophes: '''''''''''''''" str = ''''That,' she said, 'is still pointless.'''' keyä = "value1" "key\u123" = "value2" "key\U1234" = "value3" "key\k" = "value4" invInteger3 = 01 invInteger4 = 0a1234 invInteger5 = 1234_ invInteger6 = _1234 invInteger7 = 0x_1234 invInteger8 = 0o_1234 invInteger9 = 0b_1010 invInteger10 = +0x1234 invInteger11 = +0o1234 invInteger12 = +0b1010 invInteger13 = -0x1234 invInteger14 = -0o1234 invInteger15 = -0b1010 invFloat1 = .7 invFloat2 = 1. invFloat3 = 3.e4 invFloat4 = Inf invFloat5 = NaN invBool1 = True invBool2 = False )"s; toml_parser::CLexer lexer(ssOrginal); // TODO EVE //std::string ssCopy = lexer.GenerateTOML(); //EXPECT_EQ(ssOrginal, ssCopy); }