LibSQL: Database layer

This patch implements the beginnings of a database API allowing for the
creation of tables, inserting rows in those tables, and retrieving those
rows.

Tests/LibSQL/TestSqlDatabase.cpp

@@ -0,0 +1,188 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <unistd.h>
+
+#include <AK/ScopeGuard.h>
+#include <LibSQL/BTree.h>
+#include <LibSQL/Database.h>
+#include <LibSQL/Heap.h>
+#include <LibSQL/Meta.h>
+#include <LibSQL/Row.h>
+#include <LibSQL/Value.h>
+#include <LibTest/TestCase.h>
+
+NonnullRefPtr<SQL::SchemaDef> setup_schema(SQL::Database&);
+NonnullRefPtr<SQL::SchemaDef> setup_table(SQL::Database&);
+void insert_into_table(SQL::Database&, int);
+void verify_table_contents(SQL::Database&, int);
+void insert_and_verify(int);
+
+NonnullRefPtr<SQL::SchemaDef> setup_schema(SQL::Database& db)
+{
+    auto schema = SQL::SchemaDef::construct("TestSchema");
+    EXPECT(schema != nullptr);
+    db.add_schema(schema);
+    return schema;
+}
+
+NonnullRefPtr<SQL::SchemaDef> setup_table(SQL::Database& db)
+{
+    auto schema = setup_schema(db);
+    auto table = SQL::TableDef::construct(schema, "TestTable");
+    EXPECT(table != nullptr);
+    db.add_table(table);
+    table->append_column("TextColumn", SQL::SQLType::Text);
+    table->append_column("IntColumn", SQL::SQLType::Integer);
+    EXPECT_EQ(table->num_columns(), 2u);
+    db.add_table(table);
+    return table;
+}
+
+void insert_into_table(SQL::Database& db, int count)
+{
+    auto table = db.get_table("TestSchema", "TestTable");
+    EXPECT(table);
+
+    for (int ix = 0; ix < count; ix++) {
+        SQL::Row row(*table);
+        StringBuilder builder;
+        builder.appendff("Test{}", ix);
+
+        row["TextColumn"] = builder.build();
+        row["IntColumn"] = ix;
+        EXPECT(db.insert(row));
+    }
+}
+
+void verify_table_contents(SQL::Database& db, int expected_count)
+{
+    auto table = db.get_table("TestSchema", "TestTable");
+    EXPECT(table);
+
+    int sum = 0;
+    int count = 0;
+    for (auto& row : db.select_all(*table)) {
+        StringBuilder builder;
+        builder.appendff("Test{}", row["IntColumn"].to_int().value());
+        EXPECT_EQ(row["TextColumn"].to_string().value(), builder.build());
+        count++;
+        sum += row["IntColumn"].to_int().value();
+    }
+    EXPECT_EQ(count, expected_count);
+    EXPECT_EQ(sum, (expected_count * (expected_count - 1)) / 2);
+}
+
+void insert_and_verify(int count)
+{
+    ScopeGuard guard([]() { unlink("test.db"); });
+    {
+        auto db = SQL::Database::construct("test.db");
+        setup_table(db);
+        db->commit();
+    }
+    {
+        auto db = SQL::Database::construct("test.db");
+        insert_into_table(db, count);
+        db->commit();
+    }
+    {
+        auto db = SQL::Database::construct("test.db");
+        verify_table_contents(db, count);
+    }
+}
+
+TEST_CASE(create_heap)
+{
+    ScopeGuard guard([]() { unlink("test.db"); });
+    auto heap = SQL::Heap::construct("test.db");
+    EXPECT_EQ(heap->version(), 0x00000001u);
+}
+
+TEST_CASE(create_database)
+{
+    ScopeGuard guard([]() { unlink("test.db"); });
+    auto db = SQL::Database::construct("test.db");
+    db->commit();
+}
+
+TEST_CASE(add_schema_to_database)
+{
+    ScopeGuard guard([]() { unlink("test.db"); });
+    auto db = SQL::Database::construct("test.db");
+    setup_schema(db);
+    db->commit();
+}
+
+TEST_CASE(get_schema_from_database)
+{
+    ScopeGuard guard([]() { unlink("test.db"); });
+    {
+        auto db = SQL::Database::construct("test.db");
+        setup_schema(db);
+        db->commit();
+    }
+    {
+        auto db = SQL::Database::construct("test.db");
+        auto schema = db->get_schema("TestSchema");
+        EXPECT(schema);
+    }
+}
+
+TEST_CASE(add_table_to_database)
+{
+    ScopeGuard guard([]() { unlink("test.db"); });
+    auto db = SQL::Database::construct("test.db");
+    setup_table(db);
+    db->commit();
+}
+
+TEST_CASE(get_table_from_database)
+{
+    ScopeGuard guard([]() { unlink("test.db"); });
+    {
+        auto db = SQL::Database::construct("test.db");
+        setup_table(db);
+        db->commit();
+    }
+    {
+        auto db = SQL::Database::construct("test.db");
+        auto table = db->get_table("TestSchema", "TestTable");
+        EXPECT(table);
+        EXPECT_EQ(table->name(), "TestTable");
+        EXPECT_EQ(table->num_columns(), 2u);
+    }
+}
+
+TEST_CASE(insert_one_into_and_select_from_table)
+{
+    insert_and_verify(1);
+}
+
+TEST_CASE(insert_two_into_table)
+{
+    insert_and_verify(2);
+}
+
+TEST_CASE(insert_10_into_table)
+{
+    insert_and_verify(10);
+}
+
+TEST_CASE(insert_100_into_table)
+{
+    insert_and_verify(100);
+}
+
+TEST_CASE(insert_1000_into_table)
+{
+    insert_and_verify(1000);
+}
+
+TEST_CASE(insert_10000_into_table)
+{
+    insert_and_verify(10000);
+}

Userland/Libraries/LibSQL/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(SOURCES
         BTree.cpp
         BTreeIterator.cpp
+        Database.cpp
         HashIndex.cpp
         Heap.cpp
         Index.cpp
@@ -8,6 +9,7 @@ set(SOURCES
         Lexer.cpp
         Meta.cpp
         Parser.cpp
+        Row.cpp
         SyntaxHighlighter.cpp
         Token.cpp
         TreeNode.cpp

Userland/Libraries/LibSQL/Database.cpp

@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Format.h>
+#include <AK/RefPtr.h>
+#include <AK/String.h>
+
+#include <LibSQL/BTree.h>
+#include <LibSQL/Database.h>
+#include <LibSQL/Heap.h>
+#include <LibSQL/Meta.h>
+#include <LibSQL/Row.h>
+#include <LibSQL/Tuple.h>
+
+namespace SQL {
+
+Database::Database(String name)
+    : m_heap(Heap::construct(name))
+    , m_schemas(BTree::construct(*m_heap, SchemaDef::index_def()->to_tuple_descriptor(), m_heap->schemas_root()))
+    , m_tables(BTree::construct(*m_heap, TableDef::index_def()->to_tuple_descriptor(), m_heap->tables_root()))
+    , m_table_columns(BTree::construct(*m_heap, ColumnDef::index_def()->to_tuple_descriptor(), m_heap->table_columns_root()))
+{
+    m_schemas->on_new_root = [&]() {
+        m_heap->set_schemas_root(m_schemas->root());
+    };
+    m_tables->on_new_root = [&]() {
+        m_heap->set_tables_root(m_tables->root());
+    };
+    m_table_columns->on_new_root = [&]() {
+        m_heap->set_table_columns_root(m_table_columns->root());
+    };
+}
+
+void Database::add_schema(SchemaDef const& schema)
+{
+    m_schemas->insert(schema.key());
+}
+
+Key Database::get_schema_key(String const& schema_name)
+{
+    auto key = SchemaDef::make_key();
+    key["schema_name"] = schema_name;
+    return key;
+}
+
+RefPtr<SchemaDef> Database::get_schema(String const& schema_name)
+{
+    Key key = get_schema_key(schema_name);
+    auto schema_def_opt = m_schema_cache.get(key.hash());
+    if (schema_def_opt.has_value())
+        return schema_def_opt.value();
+    auto schema_iterator = m_schemas->find(key);
+    if (schema_iterator.is_end() || (*schema_iterator != key)) {
+        warnln("Schema {} not found", schema_name);
+        return nullptr;
+    }
+    auto ret = SchemaDef::construct(*schema_iterator);
+    m_schema_cache.set(key.hash(), ret);
+    return ret;
+}
+
+void Database::add_table(TableDef& table)
+{
+    m_tables->insert(table.key());
+    for (auto& column : table.columns()) {
+        m_table_columns->insert(column.key());
+    }
+}
+
+Key Database::get_table_key(String const& schema_name, String const& table_name)
+{
+    auto key = TableDef::make_key(get_schema_key(schema_name));
+    key["table_name"] = table_name;
+    return key;
+}
+
+RefPtr<TableDef> Database::get_table(String const& schema, String const& name)
+{
+    Key key = get_table_key(schema, name);
+    auto table_def_opt = m_table_cache.get(key.hash());
+    if (table_def_opt.has_value())
+        return table_def_opt.value();
+    auto table_iterator = m_tables->find(key);
+    if (table_iterator.is_end() || (*table_iterator != key)) {
+        warnln("Table {} not found", name);
+        return nullptr;
+    }
+    auto schema_def = get_schema(schema);
+    VERIFY(schema_def);
+    auto ret = TableDef::construct(schema_def, name);
+    ret->set_pointer((*table_iterator).pointer());
+    m_table_cache.set(key.hash(), ret);
+    auto hash = ret->hash();
+    auto column_key = ColumnDef::make_key(ret);
+
+    for (auto column_iterator = m_table_columns->find(column_key);
+         !column_iterator.is_end() && ((*column_iterator)["table_hash"].to_u32().value() == hash);
+         column_iterator++) {
+        ret->append_column(*column_iterator);
+    }
+    return ret;
+}
+
+Vector<Row> Database::select_all(TableDef const& table)
+{
+    VERIFY(m_table_cache.get(table.key().hash()).has_value());
+    Vector<Row> ret;
+    for (auto pointer = table.pointer(); pointer; pointer = ret.last().next_pointer()) {
+        auto buffer_or_error = m_heap->read_block(pointer);
+        if (buffer_or_error.is_error())
+            VERIFY_NOT_REACHED();
+        ret.empend(table, pointer, buffer_or_error.value());
+    }
+    return ret;
+}
+
+Vector<Row> Database::match(TableDef const& table, Key const& key)
+{
+    VERIFY(m_table_cache.get(table.key().hash()).has_value());
+    Vector<Row> ret;
+
+    // TODO Match key against indexes defined on table. If found,
+    // use the index instead of scanning the table.
+    for (auto pointer = table.pointer(); pointer;) {
+        auto buffer_or_error = m_heap->read_block(pointer);
+        if (buffer_or_error.is_error())
+            VERIFY_NOT_REACHED();
+        Row row(table, pointer, buffer_or_error.value());
+        if (row.match(key))
+            ret.append(row);
+        pointer = ret.last().next_pointer();
+    }
+    return ret;
+}
+
+bool Database::insert(Row& row)
+{
+    VERIFY(m_table_cache.get(row.table()->key().hash()).has_value());
+    row.set_pointer(m_heap->new_record_pointer());
+    row.next_pointer(row.table()->pointer());
+    update(row);
+
+    // TODO update indexes defined on table.
+
+    auto table_key = row.table()->key();
+    table_key.set_pointer(row.pointer());
+    VERIFY(m_tables->update_key_pointer(table_key));
+    row.table()->set_pointer(row.pointer());
+    return true;
+}
+
+bool Database::update(Row& tuple)
+{
+    VERIFY(m_table_cache.get(tuple.table()->key().hash()).has_value());
+    ByteBuffer buffer;
+    tuple.serialize(buffer);
+    m_heap->add_to_wal(tuple.pointer(), buffer);
+
+    // TODO update indexes defined on table.
+    return true;
+}
+
+}

Userland/Libraries/LibSQL/Database.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/RefPtr.h>
+#include <AK/String.h>
+#include <LibCore/Object.h>
+#include <LibSQL/Forward.h>
+#include <LibSQL/Heap.h>
+
+namespace SQL {
+
+/**
+ * A Database object logically connects a Heap with the SQL data we want
+ * to store in it. It has BTree pointers for B-Trees holding the definitions
+ * of tables, columns, indexes, and other SQL objects.
+ */
+class Database : public Core::Object {
+    C_OBJECT(Database);
+
+public:
+    explicit Database(String);
+    ~Database() override = default;
+
+    void commit() { m_heap->flush(); }
+
+    void add_schema(SchemaDef const&);
+    static Key get_schema_key(String const&);
+    RefPtr<SchemaDef> get_schema(String const&);
+
+    void add_table(TableDef& table);
+    static Key get_table_key(String const&, String const&);
+    RefPtr<TableDef> get_table(String const&, String const&);
+
+    Vector<Row> select_all(TableDef const&);
+    Vector<Row> match(TableDef const&, Key const&);
+    bool insert(Row&);
+    bool update(Row&);
+
+private:
+    RefPtr<Heap> m_heap;
+    RefPtr<BTree> m_schemas;
+    RefPtr<BTree> m_tables;
+    RefPtr<BTree> m_table_columns;
+
+    HashMap<u32, RefPtr<SchemaDef>> m_schema_cache;
+    HashMap<u32, RefPtr<TableDef>> m_table_cache;
+};
+
+}

Userland/Libraries/LibSQL/Forward.h

@@ -25,6 +25,7 @@ class ColumnNameExpression;
 class CommonTableExpression;
 class CommonTableExpressionList;
 class CreateTable;
+class Database;
 class TupleDescriptor;
 struct TupleElement;
 class Delete;

Userland/Libraries/LibSQL/Row.cpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibSQL/Meta.h>
+#include <LibSQL/Row.h>
+#include <LibSQL/Serialize.h>
+#include <LibSQL/Tuple.h>
+
+namespace SQL {
+
+Row::Row()
+    : Tuple()
+{
+}
+
+Row::Row(TupleDescriptor const& descriptor)
+    : Tuple(descriptor)
+{
+}
+
+Row::Row(RefPtr<TableDef> table)
+    : Tuple(table->to_tuple_descriptor())
+    , m_table(table)
+{
+}
+
+Row::Row(RefPtr<TableDef> table, u32 pointer, ByteBuffer& buffer)
+    : Tuple(table->to_tuple_descriptor())
+{
+    // FIXME Sanitize constructor situation in Tuple so this can be better
+    size_t offset = 0;
+    deserialize(buffer, offset);
+    deserialize_from<u32>(buffer, offset, m_next_pointer);
+    set_pointer(pointer);
+}
+
+void Row::serialize(ByteBuffer& buffer) const
+{
+    Tuple::serialize(buffer);
+    serialize_to<u32>(buffer, next_pointer());
+}
+
+void Row::copy_from(Row const& other)
+{
+    Tuple::copy_from(other);
+    m_next_pointer = other.next_pointer();
+}
+
+}

Userland/Libraries/LibSQL/Row.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/RefPtr.h>
+#include <LibSQL/Forward.h>
+#include <LibSQL/Value.h>
+
+namespace SQL {
+
+/**
+ * A Tuple is an element of a sequential-access persistence data structure
+ * like a flat table. Like a key it has a definition for all its parts,
+ * but unlike a key this definition is not optional.
+ *
+ * FIXME Tuples should logically belong to a TupleStore object, but right now
+ * they stand by themselves; they contain a row's worth of data and a pointer
+ * to the next Tuple.
+ */
+class Row : public Tuple {
+public:
+    Row();
+    explicit Row(TupleDescriptor const&);
+    explicit Row(RefPtr<TableDef>);
+    Row(RefPtr<TableDef>, u32, ByteBuffer&);
+    Row(Row const&) = default;
+    virtual ~Row() override = default;
+
+    [[nodiscard]] u32 next_pointer() const { return m_next_pointer; }
+    void next_pointer(u32 ptr) { m_next_pointer = ptr; }
+    RefPtr<TableDef> table() const { return m_table; }
+    virtual void serialize(ByteBuffer&) const override;
+    [[nodiscard]] virtual size_t data_length() const override { return Tuple::data_length() + sizeof(u32); }
+
+protected:
+    void copy_from(Row const&);
+
+private:
+    RefPtr<TableDef> m_table;
+    u32 m_next_pointer { 0 };
+};
+
+}