mirror of
https://github.com/eclipse-openvehicle-api/openvehicle-api.git
synced 2026-02-05 15:18:45 +00:00
1747 lines
79 KiB
C++
1747 lines
79 KiB
C++
#include <gtest/gtest.h>
|
|
#include <limits>
|
|
#include <functional>
|
|
#include "../../../sdv_services/core/toml_parser/lexer_toml.h"
|
|
#include "../../../sdv_services/core/toml_parser/exception.h"
|
|
|
|
/* Requirements Lexer
|
|
* - Lexer has to ignore all whitespaces outside of comments, basic strings and string literals
|
|
* - The Lexer has to recognize line comments
|
|
* - Control characters other than tab (U+0000 to U+0008, U+000A to U+001F, U+007F) are not permitted in comments
|
|
* - The Lexer has to recognize key-value pairs
|
|
* - with value data types
|
|
* - token_quoted_string
|
|
* - token_integer
|
|
* - token_float
|
|
* - token_boolean
|
|
* - Offset Date-Time (not included)
|
|
* - Local Date-Time (not included)
|
|
* - Local Date (not included)
|
|
* - Local Time (not included)
|
|
* - Array
|
|
* - Inline Table
|
|
* - Keys are on the left of the equals sign and values are on the right
|
|
* - Keys are always interpreted as strings
|
|
* - The key, equals sign, and value must be on the same line (though some values can be broken over multiple lines)
|
|
* - There must be a newline (or EOF) after a key-value pair. (See Inline Table for exceptions.)
|
|
* - keys
|
|
* - bare keys may contain only 'A-Za-z0-9_-'; must at least be 1 character long
|
|
* - quoted keys may be basic strings or string literals; may be empty
|
|
* - dotted keys are bare keys and dotted keys joined by a dot '.'
|
|
* - Strings
|
|
* - may only contain valid UTF-8 characters
|
|
* - Basic token_quoted_string
|
|
* - surrounded by '"'
|
|
* - chracters that must be escaped: '"', '\', and the control characters other than tab (U+0000 to U+0008, U+000A to U+001F,
|
|
U+007F)
|
|
* \b - backspace (U+0008)
|
|
\t - tab (U+0009)
|
|
\n - linefeed (U+000A)
|
|
\f - form feed (U+000C)
|
|
\r - carriage return (U+000D)
|
|
\" - quote (U+0022)
|
|
\\ - backslash (U+005C)
|
|
\uXXXX - unicode (U+XXXX)
|
|
\UXXXXXXXX - unicode (U+XXXXXXXX)
|
|
Any Unicode character may be escaped with the \uXXXX or \UXXXXXXXX forms. The escape codes must be valid Unicode scalar
|
|
values. All other escape sequences not listed above are reserved; if they are used, TOML should produce an error.
|
|
* - Multiline Basic token_quoted_string
|
|
- surrounded by three quotation marks '"""' on each side
|
|
- A newline immediately following the opening delimiter will be trimmed
|
|
- All other whitespace and newline characters remain intact
|
|
- When the last non-whitespace character on a line is an unescaped \, it will be trimmed along with all whitespace
|
|
(including newlines) up to the next non-whitespace character or closing delimiter
|
|
* - Literal
|
|
* - Mulitline Literal
|
|
*/
|
|
|
|
|
|
/*
|
|
* Requirements Lexer
|
|
* - Tokenize input
|
|
* - find identifiers (keys; bare and quoted)
|
|
* + Keys
|
|
* - find syntax tokens ([, ], =, ., {, }, newline, [[, ]]) outside of literals
|
|
* + NewLine
|
|
* + Brackets []
|
|
* + Assignment =
|
|
* + Dot .
|
|
* + Braces {}
|
|
* + TableArray [[]]
|
|
* - find and convert/unescape literals (strings, integers, floats, bools, dates, times)
|
|
* + token_integer
|
|
* + token_float
|
|
* + token_boolean
|
|
* + Strings
|
|
* - Offset Date-Time
|
|
* - Local Date-Time
|
|
* - Local Date
|
|
* - Local Time
|
|
* - ignore whitespaces outside strings
|
|
* - ignore comments
|
|
* - invalid input results in an token_error-Token
|
|
* + invalid bare key
|
|
* + invalid quoted key
|
|
* + invalid token_quoted_string
|
|
* + invalid token_integer
|
|
* + invalid token_float
|
|
* + invalid token_boolean
|
|
* - invalid Offset Date-Time
|
|
* - invalid Local Date-Time
|
|
* - invalid Local Date
|
|
* - invalid Local Time
|
|
* ? Line and Position of tokens in source file
|
|
* + Peek() returns the next token without advancing the read location
|
|
* + Peek(n) returns the next n-th token without advancing the read location
|
|
* + Consume() returns the next token and advances the read location directly after the returned token
|
|
* + Consume(n) returns the next n-th token and advances the read location directly after the returned token
|
|
* + when Peek or Consume would access a token after EOF, they return an empty token,
|
|
* if lexing was terminated, they return a TerminatedToken
|
|
* + Exceptions thrown by the CharacterReaderare caught and result in a token_terminated-Token and no further lexing will be done
|
|
*/
|
|
|
|
TEST(TOMLLexerTest, Keys)
|
|
{
|
|
using namespace std::string_literals;
|
|
// U+1F92B is the Finger-On-Lips Shushing emoji with UTF-8 byte representation 0xF09FA4AB
|
|
toml_parser::CLexer lexer(R"(
|
|
key1 = "value1"
|
|
"key\u0032" = "value2"
|
|
"key3\U0001F92B" = "abc"
|
|
'' = "valid"
|
|
'literal "key"' = "also valid"
|
|
"quoted 'key'" = "also valid"
|
|
_0xabcd = 0xabcd
|
|
1234 = 1234
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// key1, '=', value1, NewLine
|
|
// key2, '=', value2, NewLine
|
|
// key30xF09FA4AB, '=', abc, NewLine
|
|
// , '=', valid, NewLine
|
|
// literal "key", '=', also valid
|
|
const toml_parser::CToken& key1 = lexer.Consume(1);
|
|
const toml_parser::CToken& key2 = lexer.Consume(3);
|
|
const toml_parser::CToken& key3 = lexer.Consume(3);
|
|
const toml_parser::CToken& key4 = lexer.Consume(3);
|
|
const toml_parser::CToken& key5 = lexer.Consume(3);
|
|
const toml_parser::CToken& key6 = lexer.Consume(3);
|
|
const toml_parser::CToken& key7 = lexer.Consume(3);
|
|
const toml_parser::CToken& key8 = lexer.Consume(3);
|
|
ASSERT_TRUE(key1);
|
|
ASSERT_TRUE(key2);
|
|
ASSERT_TRUE(key3);
|
|
ASSERT_TRUE(key4);
|
|
ASSERT_TRUE(key5);
|
|
ASSERT_TRUE(key6);
|
|
ASSERT_TRUE(key7);
|
|
ASSERT_TRUE(key8);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, key1.Category());
|
|
EXPECT_EQ("key1", key1.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, key2.Category());
|
|
EXPECT_EQ("key\u0032", key2.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, key3.Category());
|
|
EXPECT_EQ(u8"key3\U0001F92B",key3.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, key4.Category());
|
|
EXPECT_EQ("", key4.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, key5.Category());
|
|
EXPECT_EQ("literal \"key\"", key5.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, key6.Category());
|
|
EXPECT_EQ("quoted 'key'", key6.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, key7.Category());
|
|
EXPECT_EQ("_0xabcd", key7.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, key8.Category());
|
|
EXPECT_EQ("1234", key8.StringValue());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, SyntaxToken_NewLine)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
key1 = "value"
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// key1, '=', value, NewLine
|
|
const toml_parser::CToken& newLine1 = lexer.Peek(0);
|
|
const toml_parser::CToken& newLine2 = lexer.Peek(4);
|
|
ASSERT_TRUE(newLine1);
|
|
ASSERT_TRUE(newLine2);
|
|
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, newLine1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, newLine2.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, SyntaxToken_Bracket)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
[table]
|
|
array = [[1,2],[3,4]]
|
|
emptyArray = []
|
|
[["table array"]]
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// '[', table, ']', NewLine
|
|
// array, '=', '[', '[', 1, ',', 2, ']', ',', '[', 3, ',', 4, ']', ']', NewLine
|
|
// emptyArray, '=', '[', ']', NewLine
|
|
// '[[', "table array", ']]', NewLine
|
|
const toml_parser::CToken& tableBracketOpen = lexer.Peek(1);
|
|
const toml_parser::CToken& tableBracketClose = lexer.Peek(3);
|
|
const toml_parser::CToken& arrayOfArraysBracketOpen = lexer.Peek(7);
|
|
const toml_parser::CToken& array1BracketOpen = lexer.Peek(8);
|
|
const toml_parser::CToken& array1BracketClose = lexer.Peek(12);
|
|
const toml_parser::CToken& array2BracketOpen = lexer.Peek(14);
|
|
const toml_parser::CToken& array2BracketClose = lexer.Peek(18);
|
|
const toml_parser::CToken& arrayOfArraysBracketClose = lexer.Peek(19);
|
|
const toml_parser::CToken& emptyArrayOpen = lexer.Peek(23);
|
|
const toml_parser::CToken& emptyArrayClose = lexer.Peek(24);
|
|
const toml_parser::CToken& tableArrayBracketOpen = lexer.Peek(26);
|
|
const toml_parser::CToken& tableArrayBracketClose = lexer.Peek(28);
|
|
|
|
ASSERT_TRUE(tableBracketOpen);
|
|
ASSERT_TRUE(tableBracketClose);
|
|
ASSERT_TRUE(arrayOfArraysBracketOpen);
|
|
ASSERT_TRUE(array1BracketOpen);
|
|
ASSERT_TRUE(array1BracketClose);
|
|
ASSERT_TRUE(array2BracketOpen);
|
|
ASSERT_TRUE(array2BracketClose);
|
|
ASSERT_TRUE(arrayOfArraysBracketClose);
|
|
ASSERT_TRUE(emptyArrayOpen);
|
|
ASSERT_TRUE(emptyArrayClose);
|
|
ASSERT_TRUE(tableArrayBracketOpen);
|
|
ASSERT_TRUE(tableArrayBracketClose);
|
|
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_open, tableBracketOpen.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_close, tableBracketClose.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_open, arrayOfArraysBracketOpen.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_open, array1BracketOpen.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_close, array1BracketClose.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_open, array2BracketOpen.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_close, array2BracketClose.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_close, arrayOfArraysBracketClose.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayBracketOpen.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayBracketClose.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, SyntaxToken_Assignment)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexerNormal(R"(
|
|
key = "value"
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// key, '=', value, NewLine
|
|
const toml_parser::CToken& rKeyNormal = lexerNormal.Consume(1);
|
|
const toml_parser::CToken& rAssignmentNormal = lexerNormal.Consume();
|
|
const toml_parser::CToken& rValueNormal = lexerNormal.Consume();
|
|
EXPECT_TRUE(rKeyNormal);
|
|
EXPECT_TRUE(rAssignmentNormal);
|
|
EXPECT_TRUE(rValueNormal);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNormal.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNormal.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, rValueNormal.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, rValueNormal.StringType());
|
|
EXPECT_EQ(rKeyNormal.StringValue(), "key");
|
|
EXPECT_EQ(rValueNormal.StringValue(), "value");
|
|
|
|
toml_parser::CLexer lexerNoSpace(R"(key="value")"s);
|
|
// Tokens:
|
|
// key, '=', value
|
|
const toml_parser::CToken& rKeyNoSpace = lexerNoSpace.Consume();
|
|
const toml_parser::CToken& rAssignmentNoSpace = lexerNoSpace.Consume();
|
|
const toml_parser::CToken& rValueNoSpace = lexerNoSpace.Consume();
|
|
EXPECT_TRUE(rKeyNoSpace);
|
|
EXPECT_TRUE(rAssignmentNoSpace);
|
|
EXPECT_TRUE(rValueNoSpace);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpace.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpace.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, rValueNoSpace.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, rValueNoSpace.StringType());
|
|
EXPECT_EQ(rKeyNoSpace.StringValue(), "key");
|
|
EXPECT_EQ(rValueNoSpace.StringValue(), "value");
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Comments)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexerNormal(R"(
|
|
#pre
|
|
key = "value" #follow
|
|
#next
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// key, '=', value, NewLine
|
|
const toml_parser::CToken& rKeyNormal = lexerNormal.Consume(2);
|
|
const toml_parser::CToken& rAssignmentNormal = lexerNormal.Consume();
|
|
const toml_parser::CToken& rValueNormal = lexerNormal.Consume();
|
|
EXPECT_TRUE(rKeyNormal);
|
|
EXPECT_TRUE(rAssignmentNormal);
|
|
EXPECT_TRUE(rValueNormal);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNormal.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNormal.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, rValueNormal.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, rValueNormal.StringType());
|
|
EXPECT_EQ(rKeyNormal.StringValue(), "key");
|
|
EXPECT_EQ(rValueNormal.StringValue(), "value");
|
|
|
|
toml_parser::CLexer lexerNoSpaceString(R"(key="value"#follow)"s);
|
|
// Tokens:
|
|
// key, '=', value
|
|
const toml_parser::CToken& rKeyNoSpaceString = lexerNoSpaceString.Consume();
|
|
const toml_parser::CToken& rAssignmentNoSpaceString = lexerNoSpaceString.Consume();
|
|
const toml_parser::CToken& rValueNoSpaceString = lexerNoSpaceString.Consume();
|
|
EXPECT_TRUE(rKeyNoSpaceString);
|
|
EXPECT_TRUE(rAssignmentNoSpaceString);
|
|
EXPECT_TRUE(rValueNoSpaceString);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceString.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceString.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, rValueNoSpaceString.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, rValueNoSpaceString.StringType());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceString.Next().Category());
|
|
EXPECT_EQ(rKeyNoSpaceString.StringValue(), "key");
|
|
EXPECT_EQ(rValueNoSpaceString.StringValue(), "value");
|
|
|
|
toml_parser::CLexer lexerNoSpaceInteger(R"(key=1234#follow)"s);
|
|
// Tokens:
|
|
// key, '=', value
|
|
const toml_parser::CToken& rKeyNoSpaceInteger = lexerNoSpaceInteger.Consume();
|
|
const toml_parser::CToken& rAssignmentNoSpaceInteger = lexerNoSpaceInteger.Consume();
|
|
const toml_parser::CToken& rValueNoSpaceInteger = lexerNoSpaceInteger.Consume();
|
|
EXPECT_TRUE(rKeyNoSpaceInteger);
|
|
EXPECT_TRUE(rAssignmentNoSpaceInteger);
|
|
EXPECT_TRUE(rValueNoSpaceInteger);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceInteger.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceInteger.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, rValueNoSpaceInteger.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceInteger.Next().Category());
|
|
EXPECT_EQ(rKeyNoSpaceInteger.StringValue(), "key");
|
|
EXPECT_EQ(rValueNoSpaceInteger.IntegerValue(), 1234);
|
|
|
|
toml_parser::CLexer lexerNoSpaceBoolean(R"(key=true#follow)"s);
|
|
// Tokens:
|
|
// key, '=', value
|
|
const toml_parser::CToken& rKeyNoSpaceBoolean = lexerNoSpaceBoolean.Consume();
|
|
const toml_parser::CToken& rAssignmentNoSpaceBoolean = lexerNoSpaceBoolean.Consume();
|
|
const toml_parser::CToken& rValueNoSpaceBoolean = lexerNoSpaceBoolean.Consume();
|
|
EXPECT_TRUE(rKeyNoSpaceBoolean);
|
|
EXPECT_TRUE(rAssignmentNoSpaceBoolean);
|
|
EXPECT_TRUE(rValueNoSpaceBoolean);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceBoolean.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceBoolean.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_boolean, rValueNoSpaceBoolean.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceBoolean.Next().Category());
|
|
EXPECT_EQ(rKeyNoSpaceBoolean.StringValue(), "key");
|
|
EXPECT_EQ(rValueNoSpaceBoolean.BooleanValue(), true);
|
|
|
|
toml_parser::CLexer lexerNoSpaceFloat(R"(key=123.456e78#follow)"s);
|
|
// Tokens:
|
|
// key, '=', value
|
|
const toml_parser::CToken& rKeyNoSpaceFloat = lexerNoSpaceFloat.Consume();
|
|
const toml_parser::CToken& rAssignmentNoSpaceFloat = lexerNoSpaceFloat.Consume();
|
|
const toml_parser::CToken& rValueNoSpaceFloat = lexerNoSpaceFloat.Consume();
|
|
EXPECT_TRUE(rKeyNoSpaceFloat);
|
|
EXPECT_TRUE(rAssignmentNoSpaceFloat);
|
|
EXPECT_TRUE(rValueNoSpaceFloat);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceFloat.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceFloat.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, rValueNoSpaceFloat.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceFloat.Next().Category());
|
|
EXPECT_EQ(rKeyNoSpaceFloat.StringValue(), "key");
|
|
EXPECT_EQ(rValueNoSpaceFloat.FloatValue(), 123.456e78);
|
|
|
|
toml_parser::CLexer lexerNoSpaceInlineTable(R"(key={x=1,y=2,z=3}#follow)"s);
|
|
// Tokens:
|
|
// key, '=', value
|
|
const toml_parser::CToken& rKeyNoSpaceInlineTable = lexerNoSpaceInlineTable.Consume();
|
|
const toml_parser::CToken& rAssignmentNoSpaceInlineTable = lexerNoSpaceInlineTable.Consume();
|
|
const toml_parser::CToken& rValueNoSpaceInlineTable = lexerNoSpaceInlineTable.Consume();
|
|
EXPECT_TRUE(rKeyNoSpaceInlineTable);
|
|
EXPECT_TRUE(rAssignmentNoSpaceInlineTable);
|
|
EXPECT_TRUE(rValueNoSpaceInlineTable);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceInlineTable.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceInlineTable.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, rValueNoSpaceInlineTable.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceInlineTable.Next(12).Category());
|
|
EXPECT_EQ(rKeyNoSpaceInlineTable.StringValue(), "key");
|
|
|
|
toml_parser::CLexer lexerNoSpaceArray(R"(key=[1,2,3]#follow)"s);
|
|
// Tokens:
|
|
// key, '=', value
|
|
const toml_parser::CToken& rKeyNoSpaceArray = lexerNoSpaceArray.Consume();
|
|
const toml_parser::CToken& rAssignmentNoSpaceArray = lexerNoSpaceArray.Consume();
|
|
const toml_parser::CToken& rValueNoSpaceArray = lexerNoSpaceArray.Consume();
|
|
EXPECT_TRUE(rKeyNoSpaceArray);
|
|
EXPECT_TRUE(rAssignmentNoSpaceArray);
|
|
EXPECT_TRUE(rValueNoSpaceArray);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceArray.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, rAssignmentNoSpaceArray.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_array_open, rValueNoSpaceArray.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rValueNoSpaceArray.Next(6).Category());
|
|
EXPECT_EQ(rKeyNoSpaceArray.StringValue(), "key");
|
|
|
|
toml_parser::CLexer lexerNoSpaceTable(R"([key]#follow)"s);
|
|
// Tokens:
|
|
// key, '=', value
|
|
const toml_parser::CToken& rNoSpaceTableOpen = lexerNoSpaceTable.Consume();
|
|
EXPECT_TRUE(rNoSpaceTableOpen);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_open, rNoSpaceTableOpen.Category());
|
|
const toml_parser::CToken& rKeyNoSpaceTable = lexerNoSpaceTable.Consume();
|
|
EXPECT_TRUE(rKeyNoSpaceTable);
|
|
EXPECT_EQ(rKeyNoSpaceTable.StringValue(), "key");
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, rKeyNoSpaceTable.Category());
|
|
const toml_parser::CToken& rNoSpaceTableClose = lexerNoSpaceTable.Consume();
|
|
EXPECT_TRUE(rNoSpaceTableClose);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_close, rNoSpaceTableClose.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_comment, rNoSpaceTableClose.Next().Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, SyntaxToken_Dot)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
fruit.name = "banana" # this is best practice
|
|
fruit. color = "yellow" # same as fruit.color
|
|
fruit . flavor = "banana" # same as fruit.flavor
|
|
a.b.c = 2
|
|
[dog."tater.man"]
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// fruit, '.', name, '=', banana, NewLine
|
|
// fruit, '.', color, '=', yellow, NewLine
|
|
// fruit, '.', flavor, '=', banana, NewLine
|
|
// a, '.', b, '.', c, '=', 2, NewLine
|
|
// '[', dog, '.', tater.man, ']', NewLine
|
|
const toml_parser::CToken& dot1 = lexer.Peek(2);
|
|
const toml_parser::CToken& dot2 = lexer.Peek(8);
|
|
const toml_parser::CToken& dot3 = lexer.Peek(14);
|
|
const toml_parser::CToken& dot4 = lexer.Peek(20);
|
|
const toml_parser::CToken& dot5 = lexer.Peek(22);
|
|
const toml_parser::CToken& dot6 = lexer.Peek(29);
|
|
ASSERT_TRUE(dot1);
|
|
ASSERT_TRUE(dot2);
|
|
ASSERT_TRUE(dot3);
|
|
ASSERT_TRUE(dot4);
|
|
ASSERT_TRUE(dot5);
|
|
ASSERT_TRUE(dot6);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot5.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_dot, dot6.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, SyntaxToken_Braces)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
inlineTable = { name = "Some Name", type = "Some Type" }
|
|
nestedInlineTable = { a = { a = 0, b = "b"}, e = { c = 0, d = "d"} }
|
|
emptyTable = {}
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// inlineTable, '=', '{', name, '=', Some Name, ',', type, '=', Some Type, '}', NewLine
|
|
// nestedInlineTable, '=', '{', a, '=', '{', a, '=', 0, ',' b '=', b, '}', ',' b, '=', '{', c, '=', 0, ',' d, '=', d, '}' '}',
|
|
// NewLine emptyTable, '=', '{', '}', NewLine
|
|
const toml_parser::CToken& braceOpen = lexer.Peek(3);
|
|
const toml_parser::CToken& braceClose = lexer.Peek(11);
|
|
const toml_parser::CToken& nestedOpen1 = lexer.Peek(15);
|
|
const toml_parser::CToken& nestedOpen2 = lexer.Peek(18);
|
|
const toml_parser::CToken& nestedClose2 = lexer.Peek(26);
|
|
const toml_parser::CToken& nestedOpen3 = lexer.Peek(30);
|
|
const toml_parser::CToken& nestedClose3 = lexer.Peek(38);
|
|
const toml_parser::CToken& nestedClose1 = lexer.Peek(39);
|
|
const toml_parser::CToken& emptyTableOpen = lexer.Peek(43);
|
|
const toml_parser::CToken& emptyTableClose = lexer.Peek(44);
|
|
ASSERT_TRUE(braceOpen);
|
|
ASSERT_TRUE(braceClose);
|
|
ASSERT_TRUE(nestedOpen1);
|
|
ASSERT_TRUE(nestedOpen2);
|
|
ASSERT_TRUE(nestedClose2);
|
|
ASSERT_TRUE(nestedOpen3);
|
|
ASSERT_TRUE(nestedClose3);
|
|
ASSERT_TRUE(nestedClose1);
|
|
ASSERT_TRUE(emptyTableOpen);
|
|
ASSERT_TRUE(emptyTableClose);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, braceOpen.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, braceClose.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, nestedOpen1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, nestedClose1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, nestedOpen2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, nestedClose2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, nestedOpen3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, nestedClose3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_open, emptyTableOpen.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_inline_table_close, emptyTableClose.Category());
|
|
|
|
const toml_parser::CToken& a = lexer.Peek(16);
|
|
const toml_parser::CToken& a_a = lexer.Peek(19);
|
|
const toml_parser::CToken& a_b = lexer.Peek(23);
|
|
const toml_parser::CToken& e = lexer.Peek(28);
|
|
const toml_parser::CToken& e_c = lexer.Peek(31);
|
|
const toml_parser::CToken& e_d = lexer.Peek(35);
|
|
const toml_parser::CToken& emptyTable = lexer.Peek(41);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, a.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, a_a.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, a_b.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, e.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, e_c.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, e_d.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, emptyTable.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, SyntaxToken_ArrayTable)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
[[tableArray]]
|
|
[[fruits.varieties]]
|
|
[["this is also a 'key'"]]
|
|
[['this is also a "key"']]
|
|
[[a."b.c"]]
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// '[[', tableArray, ']]', NewLine
|
|
// '[[', fruits, '.', varieties, ']]', Newline
|
|
// '[[', "this is also a 'key'", ']]', NewLine
|
|
// '[[', 'this is also a "key"', ']]', NewLine
|
|
// '[[', a, '.', "b.c", ']]', Newline
|
|
const toml_parser::CToken& tableArrayOpen1 = lexer.Peek(1);
|
|
const toml_parser::CToken& tableArrayName1 = lexer.Peek(2);
|
|
const toml_parser::CToken& tableArrayClose1 = lexer.Peek(3);
|
|
const toml_parser::CToken& tableArrayOpen2 = lexer.Peek(5);
|
|
const toml_parser::CToken& tableArrayName2a = lexer.Peek(6);
|
|
const toml_parser::CToken& tableArrayName2b = lexer.Peek(8);
|
|
const toml_parser::CToken& tableArrayClose2 = lexer.Peek(9);
|
|
const toml_parser::CToken& tableArrayOpen3 = lexer.Peek(11);
|
|
const toml_parser::CToken& tableArrayName3 = lexer.Peek(12);
|
|
const toml_parser::CToken& tableArrayClose3 = lexer.Peek(13);
|
|
const toml_parser::CToken& tableArrayOpen4 = lexer.Peek(15);
|
|
const toml_parser::CToken& tableArrayName4 = lexer.Peek(16);
|
|
const toml_parser::CToken& tableArrayClose4 = lexer.Peek(17);
|
|
const toml_parser::CToken& tableArrayOpen5 = lexer.Peek(19);
|
|
const toml_parser::CToken& tableArrayName5a = lexer.Peek(20);
|
|
const toml_parser::CToken& tableArrayName5b = lexer.Peek(22);
|
|
const toml_parser::CToken& tableArrayClose5 = lexer.Peek(23);
|
|
ASSERT_TRUE(tableArrayOpen1);
|
|
ASSERT_TRUE(tableArrayName1);
|
|
ASSERT_TRUE(tableArrayClose1);
|
|
ASSERT_TRUE(tableArrayOpen2);
|
|
ASSERT_TRUE(tableArrayName2a);
|
|
ASSERT_TRUE(tableArrayName2b);
|
|
ASSERT_TRUE(tableArrayClose2);
|
|
ASSERT_TRUE(tableArrayOpen3);
|
|
ASSERT_TRUE(tableArrayName3);
|
|
ASSERT_TRUE(tableArrayClose3);
|
|
ASSERT_TRUE(tableArrayOpen4);
|
|
ASSERT_TRUE(tableArrayName4);
|
|
ASSERT_TRUE(tableArrayClose4);
|
|
ASSERT_TRUE(tableArrayOpen5);
|
|
ASSERT_TRUE(tableArrayName5a);
|
|
ASSERT_TRUE(tableArrayName5b);
|
|
ASSERT_TRUE(tableArrayClose5);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName1.Category());
|
|
EXPECT_EQ(tableArrayName1.StringValue(), "tableArray");
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName2a.Category());
|
|
EXPECT_EQ(tableArrayName2a.StringValue(), "fruits");
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName2b.Category());
|
|
EXPECT_EQ(tableArrayName2b.StringValue(), "varieties");
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName3.Category());
|
|
EXPECT_EQ(tableArrayName3.StringValue(), "this is also a 'key'");
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName4.Category());
|
|
EXPECT_EQ(tableArrayName4.StringValue(), "this is also a \"key\"");
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_open, tableArrayOpen5.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName5a.Category());
|
|
EXPECT_EQ(tableArrayName5a.StringValue(), "a");
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, tableArrayName5b.Category());
|
|
EXPECT_EQ(tableArrayName5b.StringValue(), "b.c");
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_table_array_close, tableArrayClose5.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Datatype_Integer)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
int1 = 1
|
|
int2 = +13
|
|
int3 = -37
|
|
int4 = 1_000
|
|
int5 = 5_349_221
|
|
int6 = 53_49_221
|
|
int7 = 1_2_3_4_5
|
|
int8 = 0
|
|
int9 = +0
|
|
int10 = -0
|
|
hex1 = 0xDEADBEEF
|
|
hex2 = 0xdeadbeef
|
|
hex3 = 0xdead_beef
|
|
oct1 = 0o01234567
|
|
oct2 = 0o755
|
|
bin1 = 0b11111111
|
|
bin2 = 0b00000001
|
|
bin3 = 0b01010101
|
|
intErr1 = _1234
|
|
intErr2 = 1__234
|
|
intErr3 = 1234_
|
|
intErr4 = +0xabcd
|
|
intErr5 = -0xabcd
|
|
intErr6 = 0xabc__d
|
|
intErr7 = 0x_abcd
|
|
intErr8 = 0xabcd_
|
|
intErr9 = 0_xabcd
|
|
intErr10 = 0XDEADBEEF
|
|
)"s);
|
|
|
|
// Tokens:
|
|
// NewLine
|
|
// int1, '=', 1, Newline
|
|
// int2, '=', 13, NewLine
|
|
// int3, '=', -37, NewLine
|
|
// int4, '=', 1000, NewLine
|
|
// int5, '=', 5349221, NewLine
|
|
// int6, '=', 5349221, NewLine
|
|
// int7, '=', 12345, NewLine
|
|
// int8, '=', 0, NewLine
|
|
// int9, '=', 0, NewLine
|
|
// int10, '=', 0, NewLine
|
|
// hex1, '=', 0xDEADBEEF, NewLine
|
|
// hex2, '=', 0xDEADBEEF, NewLine
|
|
// hex3, '=', 0xDEADBEEF, NewLine
|
|
// oct1, '=', 01234567, NewLine
|
|
// oct2, '=', 0755, NewLine
|
|
// bin1, '=', 0xFF, NewLine
|
|
// bin2, '=', 0x01, NewLine
|
|
// bin3, '=', 0x55, NewLine
|
|
// intErr1, '=', _1234, NewLine
|
|
// intErr2, '=', 1__234, NewLine
|
|
// intErr3, '=', 1234_, NewLine
|
|
// intErr4, '=', +0xabcd, NewLine
|
|
// intErr5, '=', -0xabcd, NewLine
|
|
// intErr6, '=', 0xabc__d, NewLine
|
|
// intErr7, '=', 0x_abcd, NewLine
|
|
// intErr8, '=', 0xabcd_, NewLine
|
|
// intErr9, '=', 0_xabcd, NewLine
|
|
// intErr10, '=', 0XDEADBEEF, NewLine
|
|
const toml_parser::CToken& int1 = lexer.Consume(3);
|
|
const toml_parser::CToken& int2 = lexer.Consume(3);
|
|
const toml_parser::CToken& int3 = lexer.Consume(3);
|
|
const toml_parser::CToken& int4 = lexer.Consume(3);
|
|
const toml_parser::CToken& int5 = lexer.Consume(3);
|
|
const toml_parser::CToken& int6 = lexer.Consume(3);
|
|
const toml_parser::CToken& int7 = lexer.Consume(3);
|
|
const toml_parser::CToken& int8 = lexer.Consume(3);
|
|
const toml_parser::CToken& int9 = lexer.Consume(3);
|
|
const toml_parser::CToken& int10 = lexer.Consume(3);
|
|
const toml_parser::CToken& hex1 = lexer.Consume(3);
|
|
const toml_parser::CToken& hex2 = lexer.Consume(3);
|
|
const toml_parser::CToken& hex3 = lexer.Consume(3);
|
|
const toml_parser::CToken& oct1 = lexer.Consume(3);
|
|
const toml_parser::CToken& oct2 = lexer.Consume(3);
|
|
const toml_parser::CToken& bin1 = lexer.Consume(3);
|
|
const toml_parser::CToken& bin2 = lexer.Consume(3);
|
|
const toml_parser::CToken& bin3 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr1 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr2 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr3 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr4 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr5 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr6 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr7 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr8 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr9 = lexer.Consume(3);
|
|
const toml_parser::CToken& intErr10 = lexer.Consume(3);
|
|
ASSERT_TRUE(int1);
|
|
ASSERT_TRUE(int2);
|
|
ASSERT_TRUE(int3);
|
|
ASSERT_TRUE(int4);
|
|
ASSERT_TRUE(int5);
|
|
ASSERT_TRUE(int6);
|
|
ASSERT_TRUE(int7);
|
|
ASSERT_TRUE(int8);
|
|
ASSERT_TRUE(int9);
|
|
ASSERT_TRUE(int10);
|
|
ASSERT_TRUE(hex1);
|
|
ASSERT_TRUE(hex2);
|
|
ASSERT_TRUE(hex3);
|
|
ASSERT_TRUE(oct1);
|
|
ASSERT_TRUE(oct2);
|
|
ASSERT_TRUE(bin1);
|
|
ASSERT_TRUE(bin2);
|
|
ASSERT_TRUE(bin3);
|
|
ASSERT_TRUE(intErr1);
|
|
ASSERT_TRUE(intErr2);
|
|
ASSERT_TRUE(intErr3);
|
|
ASSERT_TRUE(intErr4);
|
|
ASSERT_TRUE(intErr5);
|
|
ASSERT_TRUE(intErr6);
|
|
ASSERT_TRUE(intErr7);
|
|
ASSERT_TRUE(intErr8);
|
|
ASSERT_TRUE(intErr9);
|
|
ASSERT_TRUE(intErr10);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int1.Category());
|
|
EXPECT_EQ(1, int1.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int2.Category());
|
|
EXPECT_EQ(13, int2.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int3.Category());
|
|
EXPECT_EQ(-37, int3.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int4.Category());
|
|
EXPECT_EQ(1000, int4.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int5.Category());
|
|
EXPECT_EQ(5349221, int5.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int6.Category());
|
|
EXPECT_EQ(5349221, int6.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int7.Category());
|
|
EXPECT_EQ(12345, int7.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int8.Category());
|
|
EXPECT_EQ(0, int8.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int9.Category());
|
|
EXPECT_EQ(0, int9.IntegerValue());
|
|
EXPECT_EQ(int8.IntegerValue(), int9.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, int10.Category());
|
|
EXPECT_EQ(0, int10.IntegerValue());
|
|
EXPECT_EQ(int8.IntegerValue(), int10.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, hex1.Category());
|
|
EXPECT_EQ(0xDEADBEEF, hex1.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, hex2.Category());
|
|
EXPECT_EQ(0xDEADBEEF, hex2.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, hex3.Category());
|
|
EXPECT_EQ(0xDEADBEEF, hex3.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, oct1.Category());
|
|
EXPECT_EQ(01234567, oct1.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, oct2.Category());
|
|
EXPECT_EQ(0755, oct2.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, bin1.Category());
|
|
EXPECT_EQ(0xFF, bin1.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, bin2.Category());
|
|
EXPECT_EQ(0x01, bin2.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_integer, bin3.Category());
|
|
EXPECT_EQ(0x55, bin3.IntegerValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr5.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr6.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr7.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr8.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr9.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, intErr10.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Datatype_Float)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
float1 = +1.0
|
|
float2 = 3.1415
|
|
float3 = -0.01
|
|
float4 = 5e+22
|
|
float5 = 1e06
|
|
float6 = -2E-2
|
|
float7 = 6.626e-34
|
|
float8 = 6.6_2_6e-3_4
|
|
float9 = 224_617.445_991_228
|
|
float10 = -0.0
|
|
float11 = +0.0
|
|
specialFloat12 = inf
|
|
specialFloat13 = +inf
|
|
specialFloat14 = -inf
|
|
specialFloat15 = nan
|
|
specialFloat16 = +nan
|
|
specialFloat17 = -nan
|
|
errFloat18 = 224__617.445_991_228
|
|
errFloat19 = _224_617.445_991_228
|
|
errFloat20 = 224_617_.445_991_228
|
|
errFloat21 = 224_617._445_991_228
|
|
errFloat22 = 224_617.445_991_228_
|
|
errFloat23 = 224_617.445_991__228
|
|
errFloat24 = 6.626_e-34
|
|
errFloat25 = 6.626e_-34
|
|
errFloat26 = 6.626e-_34
|
|
errFloat27 = 6.626e-34_
|
|
errFloat28 = 6.626_e-34
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// float1, '=', 1.0, Newline
|
|
// float2, '=', 3.1415, Newline
|
|
// float3, '=', -0.01, Newline
|
|
// float4, '=', 5e+22, Newline
|
|
// float5, '=', 1e+06, Newline
|
|
// float6, '=', -2e-02, Newline
|
|
// float7, '=', 6.626e-34, Newline
|
|
// float8, '=', 6.6_2_6e-3_4, Newline
|
|
// float9, '=', 224617.445991228, Newline
|
|
// float10, '=', -0.0f, Newline
|
|
// float11, '=', 0.0f, Newline
|
|
// specialFloat12, '=', inf, NewLine
|
|
// specialFloat13, '=', inf, NewLine
|
|
// specialFloat14, '=', -inf, NewLine
|
|
// specialFloat15, '=', qnan, NewLine
|
|
// specialFloat16, '=', qnan, NewLine
|
|
// specialFloat17, '=', -qnan, NewLine
|
|
// errFloat18, '=', 224__617.445_991_228, NewLine
|
|
// errFloat19, '=', _224_617.445_991_228, NewLine
|
|
// errFloat20, '=', 224_617_.445_991_228, NewLine
|
|
// errFloat21, '=', 224_617._445_991_228, NewLine
|
|
// errFloat22, '=', 224_617.445_991_228_, NewLine
|
|
// errFloat23, '=', 224_617.445_991__228, NewLine
|
|
// errFloat24, '=', 6.626_e-34, NewLine
|
|
// errFloat25, '=', 6.626e_-34, NewLine
|
|
// errFloat26, '=', 6.626e-_34, NewLine
|
|
// errFloat27, '=', 6.626e-34_, NewLine
|
|
// errFloat28, '=', 6.626_e-34, NewLine
|
|
const toml_parser::CToken& float1 = lexer.Consume(3);
|
|
const toml_parser::CToken& float2 = lexer.Consume(3);
|
|
const toml_parser::CToken& float3 = lexer.Consume(3);
|
|
const toml_parser::CToken& float4 = lexer.Consume(3);
|
|
const toml_parser::CToken& float5 = lexer.Consume(3);
|
|
const toml_parser::CToken& float6 = lexer.Consume(3);
|
|
const toml_parser::CToken& float7 = lexer.Consume(3);
|
|
const toml_parser::CToken& float8 = lexer.Consume(3);
|
|
const toml_parser::CToken& float9 = lexer.Consume(3);
|
|
const toml_parser::CToken& float10 = lexer.Consume(3);
|
|
const toml_parser::CToken& float11 = lexer.Consume(3);
|
|
const toml_parser::CToken& specialFloat12 = lexer.Consume(3);
|
|
const toml_parser::CToken& specialFloat13 = lexer.Consume(3);
|
|
const toml_parser::CToken& specialFloat14 = lexer.Consume(3);
|
|
const toml_parser::CToken& specialFloat15 = lexer.Consume(3);
|
|
const toml_parser::CToken& specialFloat16 = lexer.Consume(3);
|
|
const toml_parser::CToken& specialFloat17 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat18 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat19 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat20 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat21 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat22 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat23 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat24 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat25 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat26 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat27 = lexer.Consume(3);
|
|
const toml_parser::CToken& errFloat28 = lexer.Consume(3);
|
|
ASSERT_TRUE(float1);
|
|
ASSERT_TRUE(float2);
|
|
ASSERT_TRUE(float3);
|
|
ASSERT_TRUE(float4);
|
|
ASSERT_TRUE(float5);
|
|
ASSERT_TRUE(float6);
|
|
ASSERT_TRUE(float7);
|
|
ASSERT_TRUE(float8);
|
|
ASSERT_TRUE(float9);
|
|
ASSERT_TRUE(float10);
|
|
ASSERT_TRUE(float11);
|
|
ASSERT_TRUE(specialFloat12);
|
|
ASSERT_TRUE(specialFloat13);
|
|
ASSERT_TRUE(specialFloat14);
|
|
ASSERT_TRUE(specialFloat15);
|
|
ASSERT_TRUE(specialFloat16);
|
|
ASSERT_TRUE(specialFloat17);
|
|
ASSERT_TRUE(errFloat18);
|
|
ASSERT_TRUE(errFloat19);
|
|
ASSERT_TRUE(errFloat20);
|
|
ASSERT_TRUE(errFloat21);
|
|
ASSERT_TRUE(errFloat22);
|
|
ASSERT_TRUE(errFloat23);
|
|
ASSERT_TRUE(errFloat24);
|
|
ASSERT_TRUE(errFloat25);
|
|
ASSERT_TRUE(errFloat26);
|
|
ASSERT_TRUE(errFloat27);
|
|
ASSERT_TRUE(errFloat28);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float1.Category());
|
|
EXPECT_EQ(1.0, float1.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float2.Category());
|
|
EXPECT_EQ(3.1415, float2.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float3.Category());
|
|
EXPECT_EQ(-0.01, float3.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float4.Category());
|
|
EXPECT_EQ(5e+22, float4.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float5.Category());
|
|
EXPECT_EQ(1e+06, float5.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float6.Category());
|
|
EXPECT_EQ(-2e-02, float6.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float7.Category());
|
|
EXPECT_EQ(6.626e-34, float7.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float8.Category());
|
|
EXPECT_EQ(6.626e-34, float8.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float9.Category());
|
|
EXPECT_EQ(224617.445991228, float9.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float10.Category());
|
|
EXPECT_EQ(-0.0, float10.FloatValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, float11.Category());
|
|
EXPECT_EQ(+0.0, float11.FloatValue());
|
|
|
|
#ifdef __GNUC__
|
|
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
|
|
#endif
|
|
|
|
auto fnMakeUInt = [](double d)
|
|
{
|
|
union
|
|
{
|
|
double d;
|
|
uint64_t ui;
|
|
} temp{ d };
|
|
return temp.ui;
|
|
};
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat12.Category());
|
|
EXPECT_EQ(0x7FF0000000000000ull, fnMakeUInt(specialFloat12.FloatValue())); // inf
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat13.Category());
|
|
EXPECT_EQ(0x7FF0000000000000ull, fnMakeUInt(specialFloat13.FloatValue())); // +inf
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat14.Category());
|
|
EXPECT_EQ(0xFFF0000000000000ull, fnMakeUInt(specialFloat14.FloatValue())); // -inf
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat15.Category());
|
|
EXPECT_EQ(0x7FF0000000000000ull, fnMakeUInt(specialFloat15.FloatValue()) & 0xFFF0000000000000ull); // nan
|
|
EXPECT_NE(0, fnMakeUInt(specialFloat15.FloatValue()) & 0x000FFFFFFFFFFFFFull);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat16.Category());
|
|
EXPECT_EQ(0x7FF0000000000000ull, fnMakeUInt(specialFloat16.FloatValue()) & 0xFFF0000000000000ull);
|
|
EXPECT_NE(0, fnMakeUInt(specialFloat16.FloatValue()) & 0x000FFFFFFFFFFFFFull);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_float, specialFloat17.Category());
|
|
EXPECT_EQ(0xFFF0000000000000ull, fnMakeUInt(specialFloat17.FloatValue()) & 0xFFF0000000000000ull);
|
|
EXPECT_NE(0, fnMakeUInt(specialFloat17.FloatValue()) & 0x000FFFFFFFFFFFFFull);
|
|
#ifdef __GNUC__
|
|
#pragma GCC diagnostic pop
|
|
#endif
|
|
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat18.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat19.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat20.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat21.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat22.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat23.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat24.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat25.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat26.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat27.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, errFloat28.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Datatype_Boolean)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
key1 = true
|
|
key2 = false
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// key1, =, 1, NewLine
|
|
// key2, =, 0, NewLine
|
|
const toml_parser::CToken& trueToken = lexer.Consume(3);
|
|
const toml_parser::CToken& falseToken = lexer.Consume(3);
|
|
ASSERT_TRUE(trueToken);
|
|
ASSERT_TRUE(falseToken);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_boolean, trueToken.Category());
|
|
EXPECT_TRUE(trueToken.BooleanValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_boolean, falseToken.Category());
|
|
EXPECT_FALSE(falseToken.BooleanValue());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Datatype_String_BasicString)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
basicString = "this is a string"
|
|
basicString_esc1 = "Backspace\b"
|
|
basicString_esc2 = "Tab\t"
|
|
basicString_esc3 = "Linefeed\n"
|
|
basicString_esc4 = "Formfeed\f"
|
|
basicString_esc5 = "carriage return\r"
|
|
basicString_esc6 = "quote\""
|
|
basicString_esc7 = "backslash\\"
|
|
basicString_esc8 = "unicode hiragana 'no': \u306e"
|
|
basicString_esc9 = "Musical eighth note: \U0001D160"
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// basicString, '=', this is a string, NewLine
|
|
// basicString_esc1, '=', Bachspac, Newline
|
|
// basicString_esc2, '=', Tab\t, Newline
|
|
// basicString_esc3, '=', Linefeed\n, Newline
|
|
// basicString_esc4, '=', Formfeed\f, Newline
|
|
// basicString_esc5, '=', carriage return\r, Newline
|
|
// basicString_esc6, '=', quote", Newline
|
|
// basicString_esc7, '=', backslash\, Newline
|
|
// basicString_esc8, '=', の, Newline
|
|
// basicString_esc9, '=', 𝅘𝅥𝅮, Newline
|
|
const toml_parser::CToken& string = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc1 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc2 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc3 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc4 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc5 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc6 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc7 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc8 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc9 = lexer.Consume(3);
|
|
ASSERT_TRUE(string);
|
|
ASSERT_TRUE(string_esc1);
|
|
ASSERT_TRUE(string_esc2);
|
|
ASSERT_TRUE(string_esc3);
|
|
ASSERT_TRUE(string_esc4);
|
|
ASSERT_TRUE(string_esc5);
|
|
ASSERT_TRUE(string_esc6);
|
|
ASSERT_TRUE(string_esc7);
|
|
ASSERT_TRUE(string_esc8);
|
|
ASSERT_TRUE(string_esc9);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string.StringType());
|
|
EXPECT_EQ("this is a string", string.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc1.StringType());
|
|
EXPECT_EQ("Backspace\b", string_esc1.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc2.StringType());
|
|
EXPECT_EQ("Tab\t", string_esc2.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc3.StringType());
|
|
EXPECT_EQ("Linefeed\n", string_esc3.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc4.StringType());
|
|
EXPECT_EQ("Formfeed\f", string_esc4.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc5.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc5.StringType());
|
|
EXPECT_EQ("carriage return\r", string_esc5.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc6.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc6.StringType());
|
|
EXPECT_EQ("quote\"", string_esc6.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc7.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc7.StringType());
|
|
EXPECT_EQ("backslash\\", string_esc7.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc8.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc8.StringType());
|
|
EXPECT_EQ("unicode hiragana 'no': の", string_esc8.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc9.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, string_esc9.StringType());
|
|
EXPECT_EQ("Musical eighth note: 𝅘𝅥𝅮", string_esc9.StringValue());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Datatype_String_BasicStringMultiline)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
basicStringMultiline1 = """
|
|
this is a
|
|
multiline string"""
|
|
basicStringMultiline2 = """\
|
|
this is a nice \
|
|
|
|
multiline string"""
|
|
basicStringMultiline3 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\"."""
|
|
basicStringMultiline4 = """"This," she said, "is just a pointless statement.""""
|
|
basicStringMultiline_esc1 = """\
|
|
Backspace\b \
|
|
multiline"""
|
|
basicStringMultiline_esc2 = """\
|
|
Tab\t\
|
|
multiline"""
|
|
basicStringMultiline_esc3 = """\
|
|
Linefeed\n\
|
|
multiline"""
|
|
basicStringMultiline_esc4 = """\
|
|
Formfeed\f\
|
|
multiline"""
|
|
basicStringMultiline_esc5 = """\
|
|
carriage return\r\
|
|
multiline"""
|
|
basicStringMultiline_esc6 = """\
|
|
quote\" \
|
|
multiline"""
|
|
basicStringMultiline_esc7 = """\
|
|
backslash\\\
|
|
multiline"""
|
|
basicStringMultiline_esc8 = """\
|
|
unicode hiragana 'no': \u306e \
|
|
multiline"""
|
|
basicStringMultiline_esc9 = """\
|
|
Musical eighth note: \U0001D160 \
|
|
multiline"""
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// basicStringMultiline1, '=', this is a\nmultiline string, NewLine
|
|
// basicStringMultiline2, '=', this is a nice multiline string, NewLine
|
|
// basicStringMultiline3, '=', Here are fifteen quotation marks: """""""""""""""., NewLine
|
|
// basicStringMultiline4, '=', "This," she said, "is just a pointless statement.", NewLine
|
|
// basicStringMultiline_esc1, '=', Backspac multiline, Newline
|
|
// basicStringMultiline_esc2, '=', Tab\tmultiline, Newline
|
|
// basicStringMultiline_esc3, '=', Linefeed\nmultiline, Newline
|
|
// basicStringMultiline_esc4, '=', Formfeed\fmultiline, Newline
|
|
// basicStringMultiline_esc5, '=', carriage return\rmultiline, Newline
|
|
// basicStringMultiline_esc6, '=', quote" multiline, Newline
|
|
// basicStringMultiline_esc7, '=', backslash\multiline, Newline
|
|
// basicStringMultiline_esc8, '=', unicode hiragana 'no': の multiline, Newline
|
|
// basicStringMultiline_esc9, '=', Musical eighth note: 𝅘𝅥𝅮 multiline, Newline
|
|
const toml_parser::CToken& string1 = lexer.Consume(3);
|
|
const toml_parser::CToken& string2 = lexer.Consume(3);
|
|
const toml_parser::CToken& string3 = lexer.Consume(3);
|
|
const toml_parser::CToken& string4 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc1 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc2 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc3 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc4 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc5 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc6 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc7 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc8 = lexer.Consume(3);
|
|
const toml_parser::CToken& string_esc9 = lexer.Consume(3);
|
|
ASSERT_TRUE(string1);
|
|
ASSERT_TRUE(string2);
|
|
ASSERT_TRUE(string3);
|
|
ASSERT_TRUE(string4);
|
|
ASSERT_TRUE(string_esc1);
|
|
ASSERT_TRUE(string_esc2);
|
|
ASSERT_TRUE(string_esc3);
|
|
ASSERT_TRUE(string_esc4);
|
|
ASSERT_TRUE(string_esc5);
|
|
ASSERT_TRUE(string_esc6);
|
|
ASSERT_TRUE(string_esc7);
|
|
ASSERT_TRUE(string_esc8);
|
|
ASSERT_TRUE(string_esc9);
|
|
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string1.StringType());
|
|
EXPECT_EQ("this is a\nmultiline string", string1.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string2.StringType());
|
|
EXPECT_EQ("this is a nice multiline string", string2.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string3.StringType());
|
|
EXPECT_EQ("Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\".", string3.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string4.StringType());
|
|
EXPECT_EQ("\"This,\" she said, \"is just a pointless statement.\"", string4.StringValue());
|
|
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc1.StringType());
|
|
EXPECT_EQ("Backspace\b multiline", string_esc1.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc2.StringType());
|
|
EXPECT_EQ("Tab\tmultiline", string_esc2.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc3.StringType());
|
|
EXPECT_EQ("Linefeed\nmultiline", string_esc3.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc4.StringType());
|
|
EXPECT_EQ("Formfeed\fmultiline", string_esc4.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc5.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc5.StringType());
|
|
EXPECT_EQ("carriage return\rmultiline", string_esc5.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc6.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc6.StringType());
|
|
EXPECT_EQ("quote\" multiline", string_esc6.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc7.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc7.StringType());
|
|
EXPECT_EQ("backslash\\multiline", string_esc7.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc8.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc8.StringType());
|
|
EXPECT_EQ("unicode hiragana 'no': の multiline", string_esc8.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, string_esc9.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_quoted, string_esc9.StringType());
|
|
EXPECT_EQ("Musical eighth note: 𝅘𝅥𝅮 multiline", string_esc9.StringValue());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Datatype_String_LiteralString)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
winpath = 'C:\Users\nodejs\templates'
|
|
winpath2 = '\\ServerX\admin$\system32\'
|
|
quoted = 'Tom "Dubs" Preston-Werner'
|
|
regex = '<\i\c*\s*>'
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// winpath, '=', C:\Users\nodejs\templates, NewLine
|
|
// winpath2, '=', \\ServerX\admin$\system32\, NewLine
|
|
// quoted, '=', Tom "Dubs" Preston-Werner, NewLine
|
|
// regex, '=', <\i\c*\s*>, NewLine
|
|
|
|
const toml_parser::CToken& winpath = lexer.Consume(3);
|
|
const toml_parser::CToken& winpath2 = lexer.Consume(3);
|
|
const toml_parser::CToken& quoted = lexer.Consume(3);
|
|
const toml_parser::CToken& regex = lexer.Consume(3);
|
|
ASSERT_TRUE(winpath);
|
|
ASSERT_TRUE(winpath2);
|
|
ASSERT_TRUE(quoted);
|
|
ASSERT_TRUE(regex);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, winpath.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::literal_string, winpath.StringType());
|
|
EXPECT_EQ("C:\\Users\\nodejs\\templates", winpath.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, winpath2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::literal_string, winpath2.StringType());
|
|
EXPECT_EQ("\\\\ServerX\\admin$\\system32\\", winpath2.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, quoted.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::literal_string, quoted.StringType());
|
|
EXPECT_EQ("Tom \"Dubs\" Preston-Werner", quoted.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, regex.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::literal_string, regex.StringType());
|
|
EXPECT_EQ("<\\i\\c*\\s*>", regex.StringValue());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Datatype_String_LiteralStringMultiline)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
regex2 = '''I [dw]on't need \d{2} apples'''
|
|
lines = '''
|
|
The first newline is
|
|
trimmed in raw strings.
|
|
All other whitespace
|
|
is preserved.
|
|
'''
|
|
quot15 = '''Here are fifteen quotation marks: """""""""""""""'''
|
|
apos15 = "Here are fifteen apostrophes: '''''''''''''''"
|
|
str = ''''That,' she said, 'is still pointless.''''
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// regex2, '=', I't need \d{2} apples, NewLine
|
|
// lines, '=', The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n, NewLine
|
|
// quot15, '=', Here are fifteen quotation marks: """"""""""""""", NewLine
|
|
// apos15, '=', Here are fifteen apostropes: ''''''''''''''', NewLine
|
|
// str, '=', 'That,' she said, 'is still pointless.', NewLine
|
|
const toml_parser::CToken& regex2 = lexer.Consume(3);
|
|
const toml_parser::CToken& lines = lexer.Consume(3);
|
|
const toml_parser::CToken& quot15 = lexer.Consume(3);
|
|
const toml_parser::CToken& apos15 = lexer.Consume(3);
|
|
const toml_parser::CToken& str = lexer.Consume(3);
|
|
ASSERT_TRUE(regex2);
|
|
ASSERT_TRUE(lines);
|
|
ASSERT_TRUE(quot15);
|
|
ASSERT_TRUE(apos15);
|
|
ASSERT_TRUE(str);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, regex2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_literal, regex2.StringType());
|
|
EXPECT_EQ("I [dw]on't need \\d{2} apples", regex2.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, lines.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_literal, lines.StringType());
|
|
EXPECT_EQ("The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", lines.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, quot15.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_literal, quot15.StringType());
|
|
EXPECT_EQ("Here are fifteen quotation marks: \"\"\"\"\"\"\"\"\"\"\"\"\"\"\"", quot15.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, apos15.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::quoted_string, apos15.StringType());
|
|
EXPECT_EQ("Here are fifteen apostrophes: '''''''''''''''", apos15.StringValue());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_string, str.Category());
|
|
EXPECT_EQ(toml_parser::ETokenStringType::multi_line_literal, str.StringType());
|
|
EXPECT_EQ("'That,' she said, 'is still pointless.'", str.StringValue());
|
|
}
|
|
|
|
// TEST(TOMLLexerTest, Datatype_OffsetDateTime)
|
|
// {
|
|
// ASSERT_TRUE(false);
|
|
// }
|
|
|
|
// TEST(TOMLLexerTest, Datatype_DateTime)
|
|
// {
|
|
// ASSERT_TRUE(false);
|
|
// }
|
|
|
|
// TEST(TOMLLexerTest, Datatype_LocalDate)
|
|
// {
|
|
// ASSERT_TRUE(false);
|
|
// }
|
|
|
|
// TEST(TOMLLexerTest, Datatype_LocalTime)
|
|
// {
|
|
// ASSERT_TRUE(false);
|
|
// }
|
|
|
|
TEST(TOMLLexerTest, Invalid_Key)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
keyä = "value1"
|
|
"key\u123" = "value2"
|
|
"key\U1234" = "value3"
|
|
"key\k" = "value4"
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// ErrorToken, '=', value1, NewLine
|
|
// ErrorToken, '=', value2, NewLine
|
|
// ErrorToken, '=', value3, NewLine
|
|
// ErrorToken, '=', value4, NewLine
|
|
const toml_parser::CToken& invBareKey = lexer.Consume(1); // only [a-zA-Z0-9-_]allowed for bare key
|
|
const toml_parser::CToken& invQuotedKey1 = lexer.Consume(3); // \u has to be followed by 4 times [0-9A-Fa-F]
|
|
const toml_parser::CToken& invQuotedKey2 = lexer.Consume(3); // \U has to be followed by 8 times [0-9A-Fa-F]
|
|
const toml_parser::CToken& invQuotedKey3 = lexer.Consume(3); // any escape sequence except specified in the TOML secs are invalid
|
|
ASSERT_TRUE(invBareKey); // only [a-zA-Z0-9-_]allowed for bare key
|
|
ASSERT_TRUE(invQuotedKey1); // \u has to be followed by 4 times [0-9A-Fa-F]
|
|
ASSERT_TRUE(invQuotedKey2); // \U has to be followed by 8 times [0-9A-Fa-F]
|
|
ASSERT_TRUE(invQuotedKey3); // any escape sequence except specified in the TOML secs are invalid
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invBareKey.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invQuotedKey1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invQuotedKey2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invQuotedKey3.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Invalid_String)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer1(R"(key1 = "invalid escape sequence \h)"s);
|
|
toml_parser::CLexer lexer2(R"(key2 = """invalid escape sequence \h""")"s);
|
|
toml_parser::CLexer lexer3(R"(key3 = "\u000")"s);
|
|
toml_parser::CLexer lexer4(R"(key4 = "\U0000000")"s);
|
|
// Tokens:
|
|
// key1, '=', ErrorToken
|
|
// key2, '=', ErrorToken
|
|
// key3, '=', ErrorToken
|
|
// key4, '=', ErrorToken
|
|
const toml_parser::CToken& invString1 = lexer1.Consume(2);
|
|
const toml_parser::CToken& invString2 = lexer2.Consume(2);
|
|
const toml_parser::CToken& invString3 = lexer3.Consume(2);
|
|
const toml_parser::CToken& invString4 = lexer4.Consume(2);
|
|
ASSERT_TRUE(invString1);
|
|
ASSERT_TRUE(invString2);
|
|
ASSERT_TRUE(invString3);
|
|
ASSERT_TRUE(invString4);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invString1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invString2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invString3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invString4.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Invalid_Integer)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
invInteger3 = 01
|
|
invInteger4 = 0a1234
|
|
invInteger5 = 1234_
|
|
invInteger6 = _1234
|
|
invInteger7 = 0x_1234
|
|
invInteger8 = 0o_1234
|
|
invInteger9 = 0b_1010
|
|
invInteger10 = +0x1234
|
|
invInteger11 = +0o1234
|
|
invInteger12 = +0b1010
|
|
invInteger13 = -0x1234
|
|
invInteger14 = -0o1234
|
|
invInteger15 = -0b1010
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// invInteger3, '=', ErrorToken, NewLine
|
|
// invInteger4, '=', ErrorToken, NewLine
|
|
// invInteger5, '=', ErrorToken, NewLine
|
|
// invInteger6, '=', ErrorToken, NewLine
|
|
// invInteger7, '=', ErrorToken, NewLine
|
|
// invInteger8, '=', ErrorToken, NewLine
|
|
// invInteger9, '=', ErrorToken, NewLine
|
|
// invInteger10, '=', ErrorToken, NewLine
|
|
// invInteger11, '=', ErrorToken, NewLine
|
|
// invInteger12, '=', ErrorToken, NewLine
|
|
// invInteger12, '=', ErrorToken, NewLine
|
|
// invInteger12, '=', ErrorToken, NewLine
|
|
// invInteger12, '=', ErrorToken, NewLine
|
|
const toml_parser::CToken& invInteger3 = lexer.Consume(3); // no leading zeros
|
|
const toml_parser::CToken& invInteger4 = lexer.Consume(3); // invalid prefix
|
|
const toml_parser::CToken& invInteger5 = lexer.Consume(3); // underscores only allowed between two digits
|
|
const toml_parser::CToken& invInteger6 = lexer.Consume(3); // underscores only allowed between two digits
|
|
const toml_parser::CToken& invInteger7 = lexer.Consume(3); // underscores not allowed after prefix
|
|
const toml_parser::CToken& invInteger8 = lexer.Consume(3); // underscores not allowed after prefix
|
|
const toml_parser::CToken& invInteger9 = lexer.Consume(3); // underscores not allowed after prefix
|
|
const toml_parser::CToken& invInteger10 = lexer.Consume(3); // no leading + allowed
|
|
const toml_parser::CToken& invInteger11 = lexer.Consume(3); // no leading + allowed
|
|
const toml_parser::CToken& invInteger12 = lexer.Consume(3); // no leading + allowed
|
|
const toml_parser::CToken& invInteger13 = lexer.Consume(3); // no negative number representation allowed with hex format
|
|
const toml_parser::CToken& invInteger14 = lexer.Consume(3); // no negative number representation allowed with ocatal format
|
|
const toml_parser::CToken& invInteger15 = lexer.Consume(3); // no negative number representation allowed with binary format
|
|
ASSERT_TRUE(invInteger3); // no leading zeros
|
|
ASSERT_TRUE(invInteger4); // invalid prefix
|
|
ASSERT_TRUE(invInteger5); // underscores only allowed between two digits
|
|
ASSERT_TRUE(invInteger6); // underscores only allowed between two digits
|
|
ASSERT_TRUE(invInteger7); // underscores not allowed after prefix
|
|
ASSERT_TRUE(invInteger8); // underscores not allowed after prefix
|
|
ASSERT_TRUE(invInteger9); // underscores not allowed after prefix
|
|
ASSERT_TRUE(invInteger10); // no leading + allowed
|
|
ASSERT_TRUE(invInteger11); // no leading + allowed
|
|
ASSERT_TRUE(invInteger12); // no leading + allowed
|
|
ASSERT_TRUE(invInteger13); // no negative number representation allowed with hex format
|
|
ASSERT_TRUE(invInteger14); // no negative number representation allowed with ocatal format
|
|
ASSERT_TRUE(invInteger15); // no negative number representation allowed with binary format
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger5.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger6.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger7.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger8.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger9.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger10.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger11.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger12.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger13.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger14.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invInteger15.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Invalid_Float)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
invFloat1 = .7
|
|
invFloat2 = 1.
|
|
invFloat3 = 3.e4
|
|
invFloat4 = Inf
|
|
invFloat5 = NaN
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// invFloat1, '=', ErrorToken, NewLine
|
|
// invFloat2, '=', ErrorToken, NewLine
|
|
// invFloat3, '=', ErrorToken, NewLine
|
|
// invFloat4, '=', ErrorToken, NewLine
|
|
// invFloat5, '=', ErrorToken, NewLine
|
|
const toml_parser::CToken& invFloat1 = lexer.Consume(3); // decimal point has to be between two digits
|
|
const toml_parser::CToken& invFloat2 = lexer.Consume(3); // decimal point has to be between two digits
|
|
const toml_parser::CToken& invFloat3 = lexer.Consume(3); // decimal point has to be between two digits
|
|
const toml_parser::CToken& invFloat4 = lexer.Consume(3); // special float values are always lower case
|
|
const toml_parser::CToken& invFloat5 = lexer.Consume(3); // special float values are always lower case
|
|
ASSERT_TRUE(invFloat1); // decimal point has to be between two digits
|
|
ASSERT_TRUE(invFloat2); // decimal point has to be between two digits
|
|
ASSERT_TRUE(invFloat3); // decimal point has to be between two digits
|
|
ASSERT_TRUE(invFloat4); // special float values are always lower case
|
|
ASSERT_TRUE(invFloat5); // special float values are always lower case
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat2.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat3.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat4.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invFloat5.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Invalid_Boolean)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
invBool1 = True
|
|
invBool2 = False
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// invBool1, '=', ErrorToken, NewLine
|
|
// invBool2, '=', ErrorToken, NewLine
|
|
const toml_parser::CToken& invBool1 = lexer.Consume(3); // keyword for true has to be lower case
|
|
const toml_parser::CToken& invBool2 = lexer.Consume(3); // keyword for false has to be lower case
|
|
ASSERT_TRUE(invBool1); // keyword for true has to be lower case
|
|
ASSERT_TRUE(invBool2); // keyword for false has to be lower case
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invBool1.Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_error, invBool2.Category());
|
|
}
|
|
|
|
// TEST(TOMLLexerTest, Invalid_OffsetDateTime)
|
|
// {
|
|
// ASSERT_TRUE(false);
|
|
// }
|
|
|
|
// TEST(TOMLLexerTest, Invalid_LocalDateTime)
|
|
// {
|
|
// ASSERT_TRUE(false);
|
|
// }
|
|
|
|
// TEST(TOMLLexerTest, Invalid_LocalDate)
|
|
// {
|
|
// ASSERT_TRUE(false);
|
|
// }
|
|
|
|
// TEST(TOMLLexerTest, Invalid_LocalTime)
|
|
// {
|
|
// ASSERT_TRUE(false);
|
|
// }
|
|
|
|
TEST(TOMLLexerTest, Peek_NoAdvance)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
key1 = "value1"
|
|
key2 = "value2"
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// key1, '=', value1, NewLine
|
|
// key2, '=', value2, NewLine
|
|
ASSERT_TRUE(lexer.Peek());
|
|
ASSERT_TRUE(lexer.Peek());
|
|
ASSERT_TRUE(lexer.Peek(1));
|
|
ASSERT_TRUE(lexer.Peek(1));
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, lexer.Peek().Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, lexer.Peek().Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, lexer.Peek(1).Category());
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, lexer.Peek(1).Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, Consume_Advance)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
key1 = "value1"
|
|
key2 = "value2"
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// key1, '=', value1, NewLine
|
|
// key2, '=', value2, NewLine
|
|
auto ptr = lexer.Consume();
|
|
ASSERT_TRUE(ptr);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, ptr.Category());
|
|
ptr = lexer.Consume();
|
|
ASSERT_TRUE(ptr);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_key, ptr.Category());
|
|
ptr = lexer.Consume(2);
|
|
ASSERT_TRUE(ptr);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_new_line, ptr.Category());
|
|
ptr = lexer.Consume(1);
|
|
ASSERT_TRUE(ptr);
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_syntax_assignment, ptr.Category());
|
|
}
|
|
|
|
TEST(TOMLLexerTest, PeekConsume_BoundsCheck)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"(
|
|
key1 = "value1"
|
|
key2 = "value2"
|
|
)"s);
|
|
// Tokens:
|
|
// NewLine
|
|
// key1, '=', value1, NewLine
|
|
// key2, '=', value2, NewLine
|
|
EXPECT_FALSE(lexer.Consume(10));
|
|
EXPECT_FALSE(lexer.Consume());
|
|
EXPECT_FALSE(lexer.Peek());
|
|
EXPECT_FALSE(lexer.Peek(2));
|
|
EXPECT_FALSE(lexer.Peek(0));
|
|
EXPECT_FALSE(lexer.Consume(0));
|
|
}
|
|
|
|
TEST(TOMLLexerTest, PeekConsume_EmptyInput)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer(R"()"s);
|
|
// Tokens:
|
|
EXPECT_FALSE(lexer.Peek());
|
|
EXPECT_FALSE(lexer.Peek(1));
|
|
EXPECT_FALSE(lexer.Consume());
|
|
EXPECT_FALSE(lexer.Consume(1));
|
|
}
|
|
|
|
TEST(TOMLLexerTest, ExceptionHandling)
|
|
{
|
|
using namespace std::string_literals;
|
|
toml_parser::CLexer lexer;
|
|
try
|
|
{
|
|
lexer.Feed(std::string("") + static_cast<char>(0xC1));
|
|
}
|
|
catch (const sdv::toml::XTOMLParseException&)
|
|
{
|
|
}
|
|
|
|
// Tokens:
|
|
// token_terminated-Token
|
|
EXPECT_EQ(toml_parser::ETokenCategory::token_terminated, lexer.Consume().Category());
|
|
EXPECT_FALSE(lexer.Consume());
|
|
EXPECT_FALSE(lexer.Peek());
|
|
EXPECT_FALSE(lexer.Peek(2));
|
|
EXPECT_FALSE(lexer.Peek(0));
|
|
EXPECT_FALSE(lexer.Consume(0));
|
|
}
|
|
|
|
TEST(TOMLLexerTest, DISABLED_RegenerateTOML)
|
|
{
|
|
using namespace std::string_literals;
|
|
std::string ssOrginal = R"(
|
|
# original comments
|
|
key1 = "value1" # follow comments
|
|
"key\u0032" = "value2"
|
|
"key3\U0001F92B" = "abc"
|
|
'' = "valid"
|
|
'literal "key"' = "also valid"
|
|
"quoted 'key'" = "also valid"
|
|
_0xabcd = 0xabcd
|
|
1234 = 1234
|
|
[table]
|
|
array = [[1,2],[3,4]]
|
|
emptyArray = []
|
|
[["table array"]]
|
|
fruit.name = "banana" # this is best practice
|
|
fruit. color = "yellow" # same as fruit.color
|
|
fruit . flavor = "banana" # same as fruit.flavor
|
|
a.b.c = 2
|
|
[dog."tater.man"]
|
|
inlineTable = { name = "Some Name", type = "Some Type" }
|
|
nestedInlineTable = { a = { a = 0, b = "b"}, e = { c = 0, d = "d"} }
|
|
emptyTable = {}
|
|
[[tableArray]]
|
|
[[fruits.varieties]]
|
|
[["this is also a 'key'"]]
|
|
[['this is also a "key"']]
|
|
[[a."b.c"]]
|
|
int1 = 1
|
|
int2 = +13
|
|
int3 = -37
|
|
int4 = 1_000
|
|
int5 = 5_349_221
|
|
int6 = 53_49_221
|
|
int7 = 1_2_3_4_5
|
|
int8 = 0
|
|
int9 = +0
|
|
int10 = -0
|
|
hex1 = 0xDEADBEEF
|
|
hex2 = 0xdeadbeef
|
|
hex3 = 0xdead_beef
|
|
oct1 = 0o01234567
|
|
oct2 = 0o755
|
|
bin1 = 0b11111111
|
|
bin2 = 0b00000001
|
|
bin3 = 0b01010101
|
|
intErr1 = _1234
|
|
intErr2 = 1__234
|
|
intErr3 = 1234_
|
|
intErr4 = +0xabcd
|
|
intErr5 = -0xabcd
|
|
intErr6 = 0xabc__d
|
|
intErr7 = 0x_abcd
|
|
intErr8 = 0xabcd_
|
|
intErr9 = 0_xabcd
|
|
intErr10 = 0XDEADBEEF
|
|
float1 = +1.0
|
|
float2 = 3.1415
|
|
float3 = -0.01
|
|
float4 = 5e+22
|
|
float5 = 1e06
|
|
float6 = -2E-2
|
|
float7 = 6.626e-34
|
|
float8 = 6.6_2_6e-3_4
|
|
float9 = 224_617.445_991_228
|
|
float10 = -0.0
|
|
float11 = +0.0
|
|
specialFloat12 = inf
|
|
specialFloat13 = +inf
|
|
specialFloat14 = -inf
|
|
specialFloat15 = nan
|
|
specialFloat16 = +nan
|
|
specialFloat17 = -nan
|
|
errFloat18 = 224__617.445_991_228
|
|
errFloat19 = _224_617.445_991_228
|
|
errFloat20 = 224_617_.445_991_228
|
|
errFloat21 = 224_617._445_991_228
|
|
errFloat22 = 224_617.445_991_228_
|
|
errFloat23 = 224_617.445_991__228
|
|
errFloat24 = 6.626_e-34
|
|
errFloat25 = 6.626e_-34
|
|
errFloat26 = 6.626e-_34
|
|
errFloat27 = 6.626e-34_
|
|
errFloat28 = 6.626_e-34
|
|
key1 = true
|
|
key2 = false
|
|
basicString = "this is a string"
|
|
basicString_esc1 = "Backspace\b"
|
|
basicString_esc2 = "Tab\t"
|
|
basicString_esc3 = "Linefeed\n"
|
|
basicString_esc4 = "Formfeed\f"
|
|
basicString_esc5 = "carriage return\r"
|
|
basicString_esc6 = "quote\""
|
|
basicString_esc7 = "backslash\\"
|
|
basicString_esc8 = "unicode hiragana 'no': \u306e"
|
|
basicString_esc9 = "Musical eighth note: \U0001D160"
|
|
basicStringMultiline1 = """
|
|
this is a
|
|
multiline string"""
|
|
basicStringMultiline2 = """\
|
|
this is a nice \
|
|
|
|
multiline string"""
|
|
basicStringMultiline3 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\"."""
|
|
basicStringMultiline4 = """"This," she said, "is just a pointless statement.""""
|
|
basicStringMultiline_esc1 = """\
|
|
Backspace\b \
|
|
multiline"""
|
|
basicStringMultiline_esc2 = """\
|
|
Tab\t\
|
|
multiline"""
|
|
basicStringMultiline_esc3 = """\
|
|
Linefeed\n\
|
|
multiline"""
|
|
basicStringMultiline_esc4 = """\
|
|
Formfeed\f\
|
|
multiline"""
|
|
basicStringMultiline_esc5 = """\
|
|
carriage return\r\
|
|
multiline"""
|
|
basicStringMultiline_esc6 = """\
|
|
quote\" \
|
|
multiline"""
|
|
basicStringMultiline_esc7 = """\
|
|
backslash\\\
|
|
multiline"""
|
|
basicStringMultiline_esc8 = """\
|
|
unicode hiragana 'no': \u306e \
|
|
multiline"""
|
|
basicStringMultiline_esc9 = """\
|
|
Musical eighth note: \U0001D160 \
|
|
multiline"""
|
|
winpath = 'C:\Users\nodejs\templates'
|
|
winpath2 = '\\ServerX\admin$\system32\'
|
|
quoted = 'Tom "Dubs" Preston-Werner'
|
|
regex = '<\i\c*\s*>'
|
|
regex2 = '''I [dw]on't need \d{2} apples'''
|
|
lines = '''
|
|
The first newline is
|
|
trimmed in raw strings.
|
|
All other whitespace
|
|
is preserved.
|
|
'''
|
|
quot15 = '''Here are fifteen quotation marks: """""""""""""""'''
|
|
apos15 = "Here are fifteen apostrophes: '''''''''''''''"
|
|
str = ''''That,' she said, 'is still pointless.''''
|
|
keyä = "value1"
|
|
"key\u123" = "value2"
|
|
"key\U1234" = "value3"
|
|
"key\k" = "value4"
|
|
invInteger3 = 01
|
|
invInteger4 = 0a1234
|
|
invInteger5 = 1234_
|
|
invInteger6 = _1234
|
|
invInteger7 = 0x_1234
|
|
invInteger8 = 0o_1234
|
|
invInteger9 = 0b_1010
|
|
invInteger10 = +0x1234
|
|
invInteger11 = +0o1234
|
|
invInteger12 = +0b1010
|
|
invInteger13 = -0x1234
|
|
invInteger14 = -0o1234
|
|
invInteger15 = -0b1010
|
|
invFloat1 = .7
|
|
invFloat2 = 1.
|
|
invFloat3 = 3.e4
|
|
invFloat4 = Inf
|
|
invFloat5 = NaN
|
|
invBool1 = True
|
|
invBool2 = False
|
|
)"s;
|
|
|
|
toml_parser::CLexer lexer(ssOrginal);
|
|
|
|
// TODO EVE
|
|
//std::string ssCopy = lexer.GenerateTOML();
|
|
|
|
//EXPECT_EQ(ssOrginal, ssCopy);
|
|
}
|