/********************************************************************************
 * Copyright (c) 2025-2026 ZF Friedrichshafen AG
 *
 * This program and the accompanying materials are made available under the
 * terms of the Apache License Version 2.0 which is available at
 * https://www.apache.org/licenses/LICENSE-2.0
 *
 * SPDX-License-Identifier: Apache-2.0
 *
 * Contributors:
 *   Erik Verhoeven - initial API and implementation
 ********************************************************************************/

#include <gtest/gtest.h>
#include <limits>
#include <functional>
#include "../../../sdv_services/core/toml_parser/lexer_toml.h"
#include "../../../sdv_services/core/toml_parser/exception.h"
#include "../../../sdv_services/core/toml_parser/miscellaneous.h"

std::string DumpTokenList(toml_parser::CLexer& rLexer)
{
    auto eNavMode = rLexer.NavigationMode();
    rLexer.NavigationMode(toml_parser::CLexer::ENavigationMode::do_not_skip_anything);
    rLexer.ResetCursor();
    std::stringstream sstream;
    while (!rLexer.IsEnd())
    {
        const auto& rToken = rLexer.Consume();
        switch (rToken.Category())
        {
        case toml_parser::ETokenCategory::token_none:                           sstream << "<none>";                                break;
        case toml_parser::ETokenCategory::token_syntax_assignment:              sstream << "=";                                     break;
        case toml_parser::ETokenCategory::token_syntax_array_open:              sstream << "[";                                     break;
        case toml_parser::ETokenCategory::token_syntax_array_close:             sstream << "]";                                     break;
        case toml_parser::ETokenCategory::token_syntax_table_open:              sstream << "[";                                     break;
        case toml_parser::ETokenCategory::token_syntax_table_close:             sstream << "]";                                     break;
        case toml_parser::ETokenCategory::token_syntax_table_array_open:        sstream << "[{";                                    break;
        case toml_parser::ETokenCategory::token_syntax_table_array_close:       sstream << "}]";                                    break;
        case toml_parser::ETokenCategory::token_syntax_inline_table_open:       sstream << "{";                                     break;
        case toml_parser::ETokenCategory::token_syntax_inline_table_close:      sstream << "}";                                     break;
        case toml_parser::ETokenCategory::token_syntax_comma:                   sstream << ", ";                                    break;
        case toml_parser::ETokenCategory::token_syntax_dot:                     sstream << ".";                                     break;
        case toml_parser::ETokenCategory::token_syntax_new_line:                sstream << std::endl;                               break;
        case toml_parser::ETokenCategory::token_key:                            sstream << rToken.StringValue();                    break;
        case toml_parser::ETokenCategory::token_string:
            sstream << toml_parser::QuoteText(rToken.StringValue(),
                rToken.StringType() == toml_parser::ETokenStringType::literal_string ?
                    toml_parser::EQuoteRequest::quoted_text : toml_parser::EQuoteRequest::literal_text);
            break;
        case toml_parser::ETokenCategory::token_integer:                        sstream << rToken.IntegerValue();                   break;
        case toml_parser::ETokenCategory::token_float:                          sstream << rToken.FloatValue();                     break;
        case toml_parser::ETokenCategory::token_boolean:                        sstream << (rToken.BooleanValue() ? "true" : "false"); break;
        case toml_parser::ETokenCategory::token_time_local:                     sstream << "<time>";                                break;
        case toml_parser::ETokenCategory::token_date_time_offset:               sstream << "<date_time>";                           break;
        case toml_parser::ETokenCategory::token_date_time_local:                sstream << "<date_time>";                           break;
        case toml_parser::ETokenCategory::token_date_local:                     sstream << "<date>";                                break;
        case toml_parser::ETokenCategory::token_whitespace:                     sstream << " ";                                     break;
        case toml_parser::ETokenCategory::token_comment:                        sstream << rToken.RawString();                      break;
        case toml_parser::ETokenCategory::token_error:                          sstream << "<error>";                               break;
        case toml_parser::ETokenCategory::token_empty:                          sstream << "<empty>";                               break;
        case toml_parser::ETokenCategory::token_terminated:                     sstream << "<terminated>";                          break;
        default:                                                                sstream << "<unknown>";                             break;
        }

    }
    rLexer.ResetCursor();
    rLexer.NavigationMode(eNavMode);
    return sstream.str();
}

/**
 * @brief Helper function to determine the statemement boundaries based on a supplied key.
 * @remarks Works only when a token list and a token location has been assigned to the token.
 * @details There are assignment statements and table statements (and table array statements). 
 * @param[in] rKey Reference to the key token.
 * @return Pair of key references containing the begin and end boundary of the complete statement belonging to the key. Or returns
 * a pair of empty tokens when the statement could not be determined.
 */
std::pair<const toml_parser::CToken&, const toml_parser::CToken&> StatementBoundaries(const toml_parser::CToken& rKey)
{
    static toml_parser::CToken tokenNone;

    // Check whether the token is a valid key token.
    if (rKey.Category() != toml_parser::ETokenCategory::token_key)
        return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);
    
    // Check for a table or table array - meaning the token is preceded by a '[' or a '[[' (ignoring whitespace, newlines and
    // comments).
    std::reference_wrapper<const toml_parser::CToken> refBegin = rKey;
    std::reference_wrapper<const toml_parser::CToken> refToken = rKey;
    enum class EType {table, table_array, assignment} eType = EType::assignment;
    bool bScan = true;
    while (bScan)
    {
        refToken = refToken.get().Prev();
        if (!refToken.get()) break;
        switch (refToken.get().Category())
        {
        case toml_parser::ETokenCategory::token_syntax_table_open:
            eType = EType::table;
            refBegin = refToken;
            bScan = false;
            break;
        case toml_parser::ETokenCategory::token_syntax_table_array_open:
            eType = EType::table_array;
            refBegin = refToken;
            bScan = false;
            break;
        case toml_parser::ETokenCategory::token_key:
        case toml_parser::ETokenCategory::token_syntax_dot:
        case toml_parser::ETokenCategory::token_whitespace:
        case toml_parser::ETokenCategory::token_syntax_new_line:
        case toml_parser::ETokenCategory::token_comment:
            break;
        default:
            bScan = false;
            break;
        }
    }

    // Check for the end of the statement
    std::stack<toml_parser::ETokenCategory> stackProcessing;
    std::reference_wrapper<const toml_parser::CToken> refEnd = rKey;
    refToken = rKey;
    bScan = true;
    enum class EAssignmentState {none, key, assignment, value}
        eAssignmentState = (eType == EType::assignment ? EAssignmentState::key : EAssignmentState::none);
    while (bScan)
    {
        refToken = refToken.get().Next();
        if (!refToken.get()) break;
        switch (refToken.get().Category())
        {
        case toml_parser::ETokenCategory::token_syntax_dot:
            // When the assignment has a key followed by a dot, the assignment is actually an inline table with values.
            // Or this is not an assignment, but a parent with child table
            if (eType == EType::assignment && eAssignmentState != EAssignmentState::key)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);
            eAssignmentState = EAssignmentState::none;

            // If there is a table or table-array statement, this is not part of the parent. Re-set the start position.
            refBegin = rKey;

            // This finalizes the statement.
            bScan  = false;
            break;
        case toml_parser::ETokenCategory::token_syntax_assignment:
            // Ignore when assignments are on the stack (array or inline-table).
            if (!stackProcessing.empty())
                break;

            // Check for anything but an assignment (which is an error).
            if (eType != EType::assignment)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Check for the current state; assignment character needed
            if (eAssignmentState != EAssignmentState::key)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Set the current end and the assignment state.
            refEnd = refToken;
            eAssignmentState = EAssignmentState::assignment;
            break;
        case toml_parser::ETokenCategory::token_syntax_inline_table_open:
            // Check for anything but an assignment (which is an error).
            if (eType != EType::assignment)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Check for the current state; value assignment needed
            if (eAssignmentState != EAssignmentState::assignment)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Set the current end.
            refEnd = refToken;

            // Add the category to the stack
            stackProcessing.push(toml_parser::ETokenCategory::token_syntax_inline_table_open);
            break;
        case toml_parser::ETokenCategory::token_syntax_array_open:
            // Check for anything but an assignment (which is an error).
            if (eType != EType::assignment)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Check for the current state; value assignment needed
            if (eAssignmentState != EAssignmentState::assignment)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Set the current end.
            refEnd = refToken;

            // Add the category to the stack
            stackProcessing.push(toml_parser::ETokenCategory::token_syntax_array_open);
            break;
        case toml_parser::ETokenCategory::token_syntax_table_close:
            // Check for anything but a table.
            if (eType != EType::table)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // This finalizes the statement.
            refEnd = refToken;
            bScan  = false;
            break;
        case toml_parser::ETokenCategory::token_syntax_table_array_close:
            // Check for anything but a table array.
            if (eType != EType::table_array)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // This finalizes the statement.
            refEnd = refToken;
            bScan  = false;
            break;
        case toml_parser::ETokenCategory::token_syntax_inline_table_close:
            // Check for a fitting stack; if not existing, this is an error.
            if (stackProcessing.empty() || stackProcessing.top() != toml_parser::ETokenCategory::token_syntax_inline_table_open)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Set the current end.
            refEnd = refToken;

            // Reduce the stack and set the current token.
            stackProcessing.pop();

            // If the stack is empty, the value assignment was made
            if (stackProcessing.empty())
            {
                eAssignmentState = EAssignmentState::value;
                bScan = false;
            }
            break;
        case toml_parser::ETokenCategory::token_syntax_array_close:
            // Check for a fitting stack; if not existing, this is an error.
            if (stackProcessing.empty() || stackProcessing.top() != toml_parser::ETokenCategory::token_syntax_array_open)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Set the current end.
            refEnd = refToken;

            // Reduce the stack and set the current token.
            stackProcessing.pop();

            // If the stack is empty, the value assignment was made
            if (stackProcessing.empty())
            {
                eAssignmentState = EAssignmentState::value;
                bScan = false;
            }
            break;
        case toml_parser::ETokenCategory::token_whitespace:
        case toml_parser::ETokenCategory::token_syntax_new_line:
        case toml_parser::ETokenCategory::token_comment:
            break;
        case toml_parser::ETokenCategory::token_syntax_comma:
        case toml_parser::ETokenCategory::token_key:
            // Check for a fitting stack; if existing, this statement is valid.
            if (!stackProcessing.empty())
                break;

            // The assignment state should indicate a value.
            if (eAssignmentState != EAssignmentState::value)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Indication of the end of the statement (not including this token). 
            bScan = false;
            break;
        case toml_parser::ETokenCategory::token_string:
        case toml_parser::ETokenCategory::token_integer:
        case toml_parser::ETokenCategory::token_float:
        case toml_parser::ETokenCategory::token_boolean:
        case toml_parser::ETokenCategory::token_time_local:
        case toml_parser::ETokenCategory::token_date_time_offset:
        case toml_parser::ETokenCategory::token_date_time_local:
        case toml_parser::ETokenCategory::token_date_local:
            // Ignore when assignments are on the stack (array or inline-table).
            if (!stackProcessing.empty())
                break;

            // Check for anything but an assignment (which is an error).
            if (eType != EType::assignment)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Check for the current state; value needed.
            if (eAssignmentState != EAssignmentState::assignment)
                return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

            // Set the current end.
            refEnd = refToken;

            // The value assignment was made
            eAssignmentState = EAssignmentState::value;

            // End of the statement.
            bScan = false;
            break;
        default:
            // Invalid in this constellation
            return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);
        }
    }

    // There should be no stack entry any more...
    if (!stackProcessing.empty())
        return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

    // Assignment state should be none or value.
    if (eAssignmentState != EAssignmentState::none && eAssignmentState != EAssignmentState::value)
        return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(tokenNone, tokenNone);

    // Return the statement boundaries
    return std::pair<const toml_parser::CToken&, const toml_parser::CToken&>(refBegin.get(), refEnd.get());
}

/**
 * @brief Return the token category as string.
 * @param[in] eCategory The token category to return the text for.
 * @return The string with the category text.
 */
std::string CategoryText(toml_parser::ETokenCategory eCategory)
{
    switch (eCategory)
    {
    case toml_parser::ETokenCategory::token_none:
        return "token_none";
    case toml_parser::ETokenCategory::token_syntax_assignment:
        return "token_syntax_assignment";
    case toml_parser::ETokenCategory::token_syntax_array_open:
        return "token_syntax_array_open";
    case toml_parser::ETokenCategory::token_syntax_array_close:
        return "token_syntax_array_close";
    case toml_parser::ETokenCategory::token_syntax_table_open:
        return "token_syntax_table_open";
    case toml_parser::ETokenCategory::token_syntax_table_close:
        return "token_syntax_table_close";
    case toml_parser::ETokenCategory::token_syntax_table_array_open:
        return "token_syntax_table_array_open";
    case toml_parser::ETokenCategory::token_syntax_table_array_close:
        return "token_syntax_table_array_close";
    case toml_parser::ETokenCategory::token_syntax_inline_table_open:
        return "token_syntax_inline_table_open";
    case toml_parser::ETokenCategory::token_syntax_inline_table_close:
        return "token_syntax_inline_table_close";
    case toml_parser::ETokenCategory::token_syntax_comma:
        return "token_syntax_comma";
    case toml_parser::ETokenCategory::token_syntax_dot:
        return "token_syntax_dot";
    case toml_parser::ETokenCategory::token_syntax_new_line:
        return "token_syntax_new_line";
    case toml_parser::ETokenCategory::token_key:
        return "token_key";
    case toml_parser::ETokenCategory::token_string:
        return "token_string";
    case toml_parser::ETokenCategory::token_integer:
        return "token_integer";
    case toml_parser::ETokenCategory::token_float:
        return "token_float";
    case toml_parser::ETokenCategory::token_boolean:
        return "token_boolean";
    case toml_parser::ETokenCategory::token_time_local:
        return "token_time_local";
    case toml_parser::ETokenCategory::token_date_time_offset:
        return "token_date_time_offset";
    case toml_parser::ETokenCategory::token_date_time_local:
        return "token_date_time_local";
    case toml_parser::ETokenCategory::token_date_local:
        return "token_date_local";
    case toml_parser::ETokenCategory::token_whitespace:
        return "token_whitespace";
    case toml_parser::ETokenCategory::token_comment:
        return "token_comment";
    case toml_parser::ETokenCategory::token_error:
        return "token_error";
    case toml_parser::ETokenCategory::token_empty:
        return "token_empty";
    case toml_parser::ETokenCategory::token_terminated:
        return "token_terminated";
    default:
        return "<unknown>";
    }
}

/**
 * @brief Find routine, extend the line and check for smart boundaries
 * @param[in] rlexer Reference to the lexer to search for the tokens. The position will be reset and consumed until the
 * requested key has been found. Attention: this could be a parent key of a parent-child-composition.
 * @param[in] rssKey The key to search for.
 * @param[in] rvecCategories Reference to a vector containing the categories expected to be found.
 * @return Returns whether the key has been found and fit the list of provided categories.
 */
bool FindAndExtendToken(toml_parser::CLexer& rlexer, const std::string& rssKey,
    const std::vector<toml_parser::ETokenCategory>& rvecCategories)
{
    // Find the key
    rlexer.ResetCursor();
    while (!rlexer.IsEnd())
    {
        const toml_parser::CToken& rToken = rlexer.Consume();
        if (rToken.Category() == toml_parser::ETokenCategory::token_key && rToken.StringValue() == rssKey)
        {
            // Determine the range boundaries and create the range
            auto prBoundaries = StatementBoundaries(rToken);
            if (!prBoundaries.first || !prBoundaries.second)
            {
                std::cout << "Invalid statent boundaries..." << std::endl;
                return false;
            }
            toml_parser::CNodeTokenRange range(toml_parser::CTokenRange(prBoundaries.first, prBoundaries.second.Next()));

            // Auto extend the boundaries
            rlexer.SmartExtendNodeRange(range);
            if (!range.ExtendedNode().Begin())
            {
                std::cout << "Invalid extended node..." << std::endl;
                return false;
            }
            std::reference_wrapper<const toml_parser::CToken> refToken(range.ExtendedNode().Begin());
            size_t nIndex = 0;
            auto itCategory = rvecCategories.begin();
            do
            {
                if (nIndex == rvecCategories.size())
                {
                    std::cout << "Find " << rssKey << " Index #" << nIndex << ": Received "
                              << CategoryText(refToken.get().Category()) << std::endl;
                    return false;
                }
                if (itCategory == rvecCategories.end()) return false;
                if (*itCategory != refToken.get().Category())
                {
                    std::cout << "Find " << rssKey << " Index #" << nIndex << ": Expected " << CategoryText(*itCategory)
                              << ", received " << CategoryText(refToken.get().Category()) << std::endl;
                    return false;
                }
                nIndex++;
                itCategory++;
            } while ((refToken = refToken.get().Next()).get() && refToken.get() != range.ExtendedNode().End());
            if (itCategory != rvecCategories.end())
            {
                std::cout << "More items in the category vector than in the extended range: itCategory: "
                          << CategoryText(*itCategory) << std::endl;
                return false;
            }
            return true; // successful
        }
    }
    return false; // When coming here, the key was not found
}

TEST(TOMLLexerStatementBoundaryTests, CheckEmptyRange)
{
    std::string ssEmpty;

    // Process the code
    toml_parser::CLexer lexerEmpty(ssEmpty);

    // Check for empty range
    toml_parser::CNodeTokenRange rangeEmpty(toml_parser::CTokenRange(lexerEmpty.Consume(), lexerEmpty.Peek()));
    EXPECT_FALSE(rangeEmpty.NodeMain().Begin());
    EXPECT_FALSE(rangeEmpty.NodeMain().End());
    EXPECT_THROW(lexerEmpty.SmartExtendNodeRange(rangeEmpty), sdv::toml::XTOMLParseException);
}

TEST(TOMLLexerStatementBoundaryTests, CheckInvalidRange)
{
    std::string ssCode  = R"code(
token_a = 10
)code";
    std::string ssOther = ssCode;

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);
    toml_parser::CLexer lexerOther(ssOther);

    // Check for token of different lexing process
    toml_parser::CNodeTokenRange rangeOther(toml_parser::CTokenRange(lexerOther.Consume(), lexerOther.Peek()));
    EXPECT_TRUE(rangeOther.NodeMain().Begin());
    EXPECT_TRUE(rangeOther.NodeMain().End());
    EXPECT_THROW(lexerCode.SmartExtendNodeRange(rangeOther), sdv::toml::XTOMLParseException);
}

TEST(TOMLLexerStatementBoundaryTests, StandardIntegerAssignment)
{
    std::string ssCode  = R"code(
token_a = 10
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundaries are the space and newlines following the statement.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_a",
        {
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, StandardStringAssignment)
{
    std::string ssCode  = R"code(
token_b = "abc"
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary is the newline
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_b",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, AssignmentWithIndependentComment)
{
    std::string ssCode  = R"code(
token_c = 30.1

# middle followed by double lines


token_d = "def"
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary contains whitespace and the newline, but not the comments
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_c",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_float,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, AssignmentCommentsNotPartOfIt)
{
    std::string ssCode  = R"code(
token_c = 30.1

# middle followed by double lines


token_d = "def"

# before
# more before
token_e = [10, 20, 30]
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary doesn't contain the comments before or after, but does contain whitespace and newline.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_d",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, ArrayAssignmentWithPreceedingComment)
{
    std::string ssCode  = R"code(
token_d = "def"

# before
# more before
token_e = [10, 20, 30]

token_f = "ghi" # after
                # more after
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Array assignment. Extended boundary contains the comments before and whitespace and newline after.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_e",
        {toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_array_open,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_array_close,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, AssignmentWithFollowingComment)
{
    std::string ssCode  = R"code(
token_f = "ghi" # after
                # more after
# belonging to next
[token_g]
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary includes following comments, whitespace and newlines.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_f",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, AssignmentWithFollowingComment2)
{
    std::string ssTOML = R"code(


# This is a separate comment with several line-breaks before.
# Followed by this text on the same line.
  
# Note: there was a space after the empty comment line.
# And another separate comment on a next line.

# This is a comment before the value.
# And another comment before the value at the same line.
# 
# Note: there was a space after the empty comment line.
# But that should not influence the comment lines.
value = "this is the value text" # Comment following the value.
                                 # More comment following the value.
                                 # This becomes one line.
                                 #
                                 # But this is a separate line.

# This is also a separate comment.
# Followed by this text on the same line.

# And another text on a separate line.)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssTOML);

    // Assignment. Extended boundary includes following comments, whitespace and newlines.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "value",
        {toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, TableAssignmentWithDedicatedComment)
{
    std::string ssCode  = R"code(
token_f = "ghi" # after
                # more after
# belonging to next
[token_g]

)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Table. Extended boundary contains one comment line before (not the other two), and the whitespace and newlines after.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_g",
        {toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_table_open,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_table_close,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, AssignmentWithFollowingCommentWithoutIndentation)
{
    std::string ssCode  = R"code(
token_h = "jkl"#after without whitespace
# more after due to following empty line

[token_i]
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary includes following comments, whitespace and newlines.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_h",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, AssignmentCommentsExcluded)
{
    std::string ssCode  = R"code(
token_h = "jkl"#after without whitespace
# more after due to following empty line

[token_i]
# not after
token_j = 20
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Table. Extended boundary contains no comments and no lines after.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_i",
        {toml_parser::ETokenCategory::token_syntax_table_open,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_table_close,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, SmartExtendTokenRange)
{
    std::string ssCode  = R"code(
[token_i]
# not after
token_j = 20
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundaries include the comment before (which is not part of the previous statement) and nothing else.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_j",
        {toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, StandardAssignmentWithIndentation)
{
    std::string ssCode  = R"code(
    token_k = 30

)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundaries are the space and newlines following the statement and the space before.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_k",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, ArrayOfStringsAssignment)
{
    std::string ssCode  = R"code(
    token_l = ["abc", "def", "ghi"]
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Array assignment. Extended boundary is the newline and preceeding space.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_l",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_array_open,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_array_close,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, InlineTableAssignmentWithIndependentComment)
{
    std::string ssCode  = R"code(
    token_m = {x = 10, str = "gfh"}

    # middle followed by double newlines


    token_n = 100
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Inline table. Extended boundary contains whitespace and the newline, but not the comments; preceeding and following.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_m",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_inline_table_open,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_inline_table_close,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, IndentedAssignmentNoComments)
{
    std::string ssCode  = R"code(
    # begin followed by double newlines


    token_n = 100

    # before
    # more before
    token_o = 123.456
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary doesn't contain the comments before or after, but does contain whitespace and newlines.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_n",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, IndentedAssignmentWithCommentsBefore)
{
    std::string ssCode  = R"code(

    # before
    # more before
    token_o = 123.456

)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary contains the comments and whitespace before and newline after.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_o",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_float,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, IndentedBooleanArrayAssignmentWithDedicatedComments)
{
    std::string ssCode  = R"code(

    token_p = [true, false] # after
            # more after
    # belonging to next
    token_q = "next"
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Array assignment. Extended boundary includes following comments, whitespace and newlines and preceeding whitespace.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_p",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_array_open,
            toml_parser::ETokenCategory::token_boolean,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_boolean,
            toml_parser::ETokenCategory::token_syntax_array_close,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, IndentedAssignmentWithDedicatedCommentsBefore)
{
    std::string ssCode  = R"code(
    token_p = [true, false] # after
            # more after
    # belonging to next
    token_q = "next"

)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary contains one comment line before (not the other two), and the whitespace and newlines after.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_q",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, IndentedAssignmentFollowedByCommentWithoutSpace)
{
    std::string ssCode  = R"code(
    token_r =987#after without whitespace
    # more after due to following empty line

    [token_s]
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Assignment. Extended boundary includes following comments, whitespace and newlines.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_r",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, IndentedTableWithoutComments)
{
    std::string ssCode  = R"code(
    [token_s]
    # not after
    [token_t]
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Table. Extended boundary contains no comments and no lines after, but whitespace before.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_s",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_table_open,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_table_close,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, IndentedTableWithAndWithoutComments)
{
    std::string ssCode  = R"code(
    [token_s]
    # not after
    [token_t]

# comment before a table member (belongs to token_bb)
token_u.token_aa.token_bb = 10     # table token_u has a table token_aa which has a value token_bb
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Table. Extended boundaries include the comment before (which is not part of the previous statement) and nothing else.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_t",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_table_open,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_table_close,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, InlineTableWithParentChildAssignment)
{
    std::string ssCode  = R"code(
# comment before a table member (belongs to token_bb)
token_u.token_aa.token_bb = 10     # table token_u has a table token_aa which has a value token_bb
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Table value. Extended boundaries include the dot.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_u",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_dot}));
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_aa",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_dot,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_dot}));
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_bb",
        {toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_dot,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_dot,
            toml_parser::ETokenCategory::token_key, 
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, ComplexInlineTableWithParentChildAssignmentOfTablesAndMultiDimensionalArrays)
{
    std::string ssCode  = R"code(
# Super inline table with sub-tables and arrays
token_u.token_aa.token_cc = { dd = { ee = 10, ff = 11 },    # this is the comment for dd
                              gg = [{hh = 1, ii = 2},
                                    {hh = 3, ii = 4},
                                    {hh = 5},
                                    {ii = 6}],             # this is the comment for gg
                                                           # and this as well
                              jj = [["abc", "def"], [1, 2, 3], []]}

)code";

    // NOTE EVE 22.10.2025: the value token_u.token_aa.token_cc.jj is the last value in the table. This means that the scope of
    // the value included the comma before the value jj (following value gg). This has the consequence, that the comments, which
    // actually belong to ggh are also part of this and if deleted or inserted before, the comments are at the wrong location.
    // This currently is by design and cannot be changed so easily (needs a different type of parser).

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Complex table value with multiple inline tables and multi-dimensional arrays.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "ee",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma}));
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "ff",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_whitespace}));
     EXPECT_TRUE(FindAndExtendToken(lexerCode, "dd",
         {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_inline_table_open,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_inline_table_close,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line}));
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "hh",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma}));
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "ii",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer}));
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "gg",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,                         // gg
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_array_open,
            toml_parser::ETokenCategory::token_syntax_inline_table_open,
            toml_parser::ETokenCategory::token_key,                         // hh = 1
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,                         // ii = 2
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_inline_table_close,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_inline_table_open,
            toml_parser::ETokenCategory::token_key,                         // hh = 3
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,                         // ii = 4
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_inline_table_close,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_inline_table_open,
            toml_parser::ETokenCategory::token_key,                         // hh = 5
            toml_parser::ETokenCategory::token_whitespace,  
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_inline_table_close,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_inline_table_open,
            toml_parser::ETokenCategory::token_key,                         // ii = 6
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_inline_table_close,
            toml_parser::ETokenCategory::token_syntax_array_close,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line}));
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "jj",
        {toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_assignment,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_array_open,
            toml_parser::ETokenCategory::token_syntax_array_open,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_string,
            toml_parser::ETokenCategory::token_syntax_array_close,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_array_open,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_integer,
            toml_parser::ETokenCategory::token_syntax_array_close,
            toml_parser::ETokenCategory::token_syntax_comma,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_syntax_array_open,
            toml_parser::ETokenCategory::token_syntax_array_close,
            toml_parser::ETokenCategory::token_syntax_array_close}));
}

TEST(TOMLLexerStatementBoundaryTests, ArrayOfTables)
{
    std::string ssCode  = R"code(
[[token_v]]
token_kk = 10
token_ll = 20
[token_w]
token_mm = 30
[[token_v]]
token_kk = 110
token_ll = 120
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Table of arrays.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_v",
        {toml_parser::ETokenCategory::token_syntax_table_array_open,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_table_array_close,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}

TEST(TOMLLexerStatementBoundaryTests, ParentChildTable)
{
    std::string ssCode  = R"code(
[[token_v]]
token_kk = 110
token_ll = 120
# Comments before (belongs ot child only)
[token_x.token_nn] # Comments following (belongs to child only)
)code";

    // Process the code
    toml_parser::CLexer lexerCode(ssCode);

    // Table in implcit table.
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_x",
        {toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_dot}));
    EXPECT_TRUE(FindAndExtendToken(lexerCode, "token_nn",
        {toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line,
            toml_parser::ETokenCategory::token_syntax_table_open,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_dot,
            toml_parser::ETokenCategory::token_key,
            toml_parser::ETokenCategory::token_syntax_table_close,
            toml_parser::ETokenCategory::token_whitespace,
            toml_parser::ETokenCategory::token_comment,
            toml_parser::ETokenCategory::token_syntax_new_line}));
}