ladybird/Userland/Libraries/LibSQL/Lexer.cpp

/*
 * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "Lexer.h"
#include <AK/Debug.h>
#include <ctype.h>

namespace SQL {

HashMap<String, TokenType> Lexer::s_keywords;
HashMap<char, TokenType> Lexer::s_one_char_tokens;
HashMap<String, TokenType> Lexer::s_two_char_tokens;

Lexer::Lexer(StringView source)
    : m_source(source)
{
    if (s_keywords.is_empty()) {
#define __ENUMERATE_SQL_TOKEN(value, type, category)       \
    if (TokenCategory::category == TokenCategory::Keyword) \
        s_keywords.set(value, TokenType::type);
        ENUMERATE_SQL_TOKENS
#undef __ENUMERATE_SQL_TOKEN
    }

    if (s_one_char_tokens.is_empty()) {
#define __ENUMERATE_SQL_TOKEN(value, type, category)                                          \
    if (TokenCategory::category != TokenCategory::Keyword && StringView(value).length() == 1) \
        s_one_char_tokens.set(value[0], TokenType::type);
        ENUMERATE_SQL_TOKENS
#undef __ENUMERATE_SQL_TOKEN
    }

    if (s_two_char_tokens.is_empty()) {
#define __ENUMERATE_SQL_TOKEN(value, type, category)                                          \
    if (TokenCategory::category != TokenCategory::Keyword && StringView(value).length() == 2) \
        s_two_char_tokens.set(value, TokenType::type);
        ENUMERATE_SQL_TOKENS
#undef __ENUMERATE_SQL_TOKEN
    }

    consume();
}

Token Lexer::next()
{
    bool found_invalid_comment = consume_whitespace_and_comments();

    size_t value_start = m_position;
    size_t value_start_line_number = m_line_number;
    size_t value_start_column_number = m_line_column;
    auto token_type = TokenType::Invalid;

    if (is_eof()) {
        token_type = found_invalid_comment ? TokenType::Invalid : TokenType::Eof;
    } else if (is_identifier_start()) {
        do {
            consume();
        } while (is_identifier_middle());

        if (auto it = s_keywords.find(m_source.substring_view(value_start - 1, m_position - value_start)); it != s_keywords.end()) {
            token_type = it->value;
        } else {
            token_type = TokenType::Identifier;
        }
    } else if (is_numeric_literal_start()) {
        token_type = TokenType::NumericLiteral;
        if (!consume_numeric_literal())
            token_type = TokenType::Invalid;
    } else {
        bool found_two_char_token = false;
        if (m_position < m_source.length()) {
            if (auto it = s_two_char_tokens.find(m_source.substring_view(m_position - 1, 2)); it != s_two_char_tokens.end()) {
                found_two_char_token = true;
                token_type = it->value;
                consume();
                consume();
            }
        }

        bool found_one_char_token = false;
        if (!found_two_char_token) {
            if (auto it = s_one_char_tokens.find(m_current_char); it != s_one_char_tokens.end()) {
                found_one_char_token = true;
                token_type = it->value;
                consume();
            }
        }

        if (!found_two_char_token && !found_one_char_token) {
            token_type = TokenType::Invalid;
            consume();
        }
    }

    Token token(token_type, m_source.substring_view(value_start - 1, m_position - value_start), value_start_line_number, value_start_column_number);

    if constexpr (SQL_DEBUG) {
        dbgln("------------------------------");
        dbgln("Token: {}", token.name());
        dbgln("Value: {}", token.value());
        dbgln("Line: {}, Column: {}", token.line_number(), token.line_column());
        dbgln("------------------------------");
    }

    return token;
}

void Lexer::consume()
{
    auto did_reach_eof = [this] {
        if (m_position != m_source.length())
            return false;
        m_current_char = EOF;
        ++m_line_column;
        ++m_position;
        return true;
    };

    if (m_position > m_source.length())
        return;

    if (did_reach_eof())
        return;

    if (is_line_break()) {
        ++m_line_number;
        m_line_column = 1;
    } else {
        ++m_line_column;
    }

    m_current_char = m_source[m_position++];
}

bool Lexer::consume_whitespace_and_comments()
{
    bool found_invalid_comment = false;

    while (true) {
        if (isspace(m_current_char)) {
            do {
                consume();
            } while (isspace(m_current_char));
        } else if (is_line_comment_start()) {
            consume();
            do {
                consume();
            } while (!is_eof() && !is_line_break());
        } else if (is_block_comment_start()) {
            consume();
            do {
                consume();
            } while (!is_eof() && !is_block_comment_end());
            if (is_eof())
                found_invalid_comment = true;
            consume(); // consume *
            if (is_eof())
                found_invalid_comment = true;
            consume(); // consume /
        } else {
            break;
        }
    }

    return found_invalid_comment;
}

bool Lexer::consume_numeric_literal()
{
    // https://www.sqlite.org/draft/syntax/numeric-literal.html
    bool is_valid_numeric_literal = true;

    if (m_current_char == '0') {
        consume();
        if (m_current_char == '.') {
            consume();
            while (isdigit(m_current_char))
                consume();
            if (m_current_char == 'e' || m_current_char == 'E')
                is_valid_numeric_literal = consume_exponent();
        } else if (m_current_char == 'e' || m_current_char == 'E') {
            is_valid_numeric_literal = consume_exponent();
        } else if (m_current_char == 'x' || m_current_char == 'X') {
            is_valid_numeric_literal = consume_hexadecimal_number();
        } else if (isdigit(m_current_char)) {
            do {
                consume();
            } while (isdigit(m_current_char));
        }
    } else {
        do {
            consume();
        } while (isdigit(m_current_char));

        if (m_current_char == '.') {
            consume();
            while (isdigit(m_current_char))
                consume();
        }
        if (m_current_char == 'e' || m_current_char == 'E')
            is_valid_numeric_literal = consume_exponent();
    }

    return is_valid_numeric_literal;
}

bool Lexer::consume_exponent()
{
    consume();
    if (m_current_char == '-' || m_current_char == '+')
        consume();

    if (!isdigit(m_current_char))
        return false;

    while (isdigit(m_current_char)) {
        consume();
    }
    return true;
}

bool Lexer::consume_hexadecimal_number()
{
    consume();
    if (!isxdigit(m_current_char))
        return false;

    while (isxdigit(m_current_char))
        consume();

    return true;
}

bool Lexer::match(char a, char b) const
{
    if (m_position >= m_source.length())
        return false;

    return m_current_char == a
        && m_source[m_position] == b;
}

bool Lexer::is_identifier_start() const
{
    return isalpha(m_current_char) || m_current_char == '_';
}

bool Lexer::is_identifier_middle() const
{
    return is_identifier_start() || isdigit(m_current_char);
}

bool Lexer::is_numeric_literal_start() const
{
    return isdigit(m_current_char) || (m_current_char == '.' && m_position < m_source.length() && isdigit(m_source[m_position]));
}

bool Lexer::is_line_comment_start() const
{
    return match('-', '-');
}

bool Lexer::is_block_comment_start() const
{
    return match('/', '*');
}

bool Lexer::is_block_comment_end() const
{
    return match('*', '/');
}

bool Lexer::is_line_break() const
{
    return m_current_char == '\n';
}

bool Lexer::is_eof() const
{
    return m_current_char == EOF;
}

}