From 87bd69559fd5c63564fee853a5d93d4acc0023c8 Mon Sep 17 00:00:00 2001 From: Jan de Visser Date: Thu, 17 Jun 2021 14:12:46 -0400 Subject: [PATCH] LibSQL: Database layer This patch implements the beginnings of a database API allowing for the creation of tables, inserting rows in those tables, and retrieving those rows. --- Tests/LibSQL/TestSqlDatabase.cpp | 188 +++++++++++++++++++++++ Userland/Libraries/LibSQL/CMakeLists.txt | 2 + Userland/Libraries/LibSQL/Database.cpp | 166 ++++++++++++++++++++ Userland/Libraries/LibSQL/Database.h | 54 +++++++ Userland/Libraries/LibSQL/Forward.h | 1 + Userland/Libraries/LibSQL/Row.cpp | 52 +++++++ Userland/Libraries/LibSQL/Row.h | 48 ++++++ 7 files changed, 511 insertions(+) create mode 100644 Tests/LibSQL/TestSqlDatabase.cpp create mode 100644 Userland/Libraries/LibSQL/Database.cpp create mode 100644 Userland/Libraries/LibSQL/Database.h create mode 100644 Userland/Libraries/LibSQL/Row.cpp create mode 100644 Userland/Libraries/LibSQL/Row.h diff --git a/Tests/LibSQL/TestSqlDatabase.cpp b/Tests/LibSQL/TestSqlDatabase.cpp new file mode 100644 index 00000000000..de02a75857c --- /dev/null +++ b/Tests/LibSQL/TestSqlDatabase.cpp @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2021, Jan de Visser + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +NonnullRefPtr setup_schema(SQL::Database&); +NonnullRefPtr setup_table(SQL::Database&); +void insert_into_table(SQL::Database&, int); +void verify_table_contents(SQL::Database&, int); +void insert_and_verify(int); + +NonnullRefPtr setup_schema(SQL::Database& db) +{ + auto schema = SQL::SchemaDef::construct("TestSchema"); + EXPECT(schema != nullptr); + db.add_schema(schema); + return schema; +} + +NonnullRefPtr setup_table(SQL::Database& db) +{ + auto schema = setup_schema(db); + auto table = SQL::TableDef::construct(schema, "TestTable"); + EXPECT(table != nullptr); + db.add_table(table); + table->append_column("TextColumn", SQL::SQLType::Text); + table->append_column("IntColumn", SQL::SQLType::Integer); + EXPECT_EQ(table->num_columns(), 2u); + db.add_table(table); + return table; +} + +void insert_into_table(SQL::Database& db, int count) +{ + auto table = db.get_table("TestSchema", "TestTable"); + EXPECT(table); + + for (int ix = 0; ix < count; ix++) { + SQL::Row row(*table); + StringBuilder builder; + builder.appendff("Test{}", ix); + + row["TextColumn"] = builder.build(); + row["IntColumn"] = ix; + EXPECT(db.insert(row)); + } +} + +void verify_table_contents(SQL::Database& db, int expected_count) +{ + auto table = db.get_table("TestSchema", "TestTable"); + EXPECT(table); + + int sum = 0; + int count = 0; + for (auto& row : db.select_all(*table)) { + StringBuilder builder; + builder.appendff("Test{}", row["IntColumn"].to_int().value()); + EXPECT_EQ(row["TextColumn"].to_string().value(), builder.build()); + count++; + sum += row["IntColumn"].to_int().value(); + } + EXPECT_EQ(count, expected_count); + EXPECT_EQ(sum, (expected_count * (expected_count - 1)) / 2); +} + +void insert_and_verify(int count) +{ + ScopeGuard guard([]() { unlink("test.db"); }); + { + auto db = SQL::Database::construct("test.db"); + setup_table(db); + db->commit(); + } + { + auto db = SQL::Database::construct("test.db"); + insert_into_table(db, count); + db->commit(); + } + { + auto db = SQL::Database::construct("test.db"); + verify_table_contents(db, count); + } +} + +TEST_CASE(create_heap) +{ + ScopeGuard guard([]() { unlink("test.db"); }); + auto heap = SQL::Heap::construct("test.db"); + EXPECT_EQ(heap->version(), 0x00000001u); +} + +TEST_CASE(create_database) +{ + ScopeGuard guard([]() { unlink("test.db"); }); + auto db = SQL::Database::construct("test.db"); + db->commit(); +} + +TEST_CASE(add_schema_to_database) +{ + ScopeGuard guard([]() { unlink("test.db"); }); + auto db = SQL::Database::construct("test.db"); + setup_schema(db); + db->commit(); +} + +TEST_CASE(get_schema_from_database) +{ + ScopeGuard guard([]() { unlink("test.db"); }); + { + auto db = SQL::Database::construct("test.db"); + setup_schema(db); + db->commit(); + } + { + auto db = SQL::Database::construct("test.db"); + auto schema = db->get_schema("TestSchema"); + EXPECT(schema); + } +} + +TEST_CASE(add_table_to_database) +{ + ScopeGuard guard([]() { unlink("test.db"); }); + auto db = SQL::Database::construct("test.db"); + setup_table(db); + db->commit(); +} + +TEST_CASE(get_table_from_database) +{ + ScopeGuard guard([]() { unlink("test.db"); }); + { + auto db = SQL::Database::construct("test.db"); + setup_table(db); + db->commit(); + } + { + auto db = SQL::Database::construct("test.db"); + auto table = db->get_table("TestSchema", "TestTable"); + EXPECT(table); + EXPECT_EQ(table->name(), "TestTable"); + EXPECT_EQ(table->num_columns(), 2u); + } +} + +TEST_CASE(insert_one_into_and_select_from_table) +{ + insert_and_verify(1); +} + +TEST_CASE(insert_two_into_table) +{ + insert_and_verify(2); +} + +TEST_CASE(insert_10_into_table) +{ + insert_and_verify(10); +} + +TEST_CASE(insert_100_into_table) +{ + insert_and_verify(100); +} + +TEST_CASE(insert_1000_into_table) +{ + insert_and_verify(1000); +} + +TEST_CASE(insert_10000_into_table) +{ + insert_and_verify(10000); +} diff --git a/Userland/Libraries/LibSQL/CMakeLists.txt b/Userland/Libraries/LibSQL/CMakeLists.txt index 5c8ac80d435..0e979d69f8f 100644 --- a/Userland/Libraries/LibSQL/CMakeLists.txt +++ b/Userland/Libraries/LibSQL/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCES BTree.cpp BTreeIterator.cpp + Database.cpp HashIndex.cpp Heap.cpp Index.cpp @@ -8,6 +9,7 @@ set(SOURCES Lexer.cpp Meta.cpp Parser.cpp + Row.cpp SyntaxHighlighter.cpp Token.cpp TreeNode.cpp diff --git a/Userland/Libraries/LibSQL/Database.cpp b/Userland/Libraries/LibSQL/Database.cpp new file mode 100644 index 00000000000..0ff90f6257f --- /dev/null +++ b/Userland/Libraries/LibSQL/Database.cpp @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2021, Jan de Visser + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace SQL { + +Database::Database(String name) + : m_heap(Heap::construct(name)) + , m_schemas(BTree::construct(*m_heap, SchemaDef::index_def()->to_tuple_descriptor(), m_heap->schemas_root())) + , m_tables(BTree::construct(*m_heap, TableDef::index_def()->to_tuple_descriptor(), m_heap->tables_root())) + , m_table_columns(BTree::construct(*m_heap, ColumnDef::index_def()->to_tuple_descriptor(), m_heap->table_columns_root())) +{ + m_schemas->on_new_root = [&]() { + m_heap->set_schemas_root(m_schemas->root()); + }; + m_tables->on_new_root = [&]() { + m_heap->set_tables_root(m_tables->root()); + }; + m_table_columns->on_new_root = [&]() { + m_heap->set_table_columns_root(m_table_columns->root()); + }; +} + +void Database::add_schema(SchemaDef const& schema) +{ + m_schemas->insert(schema.key()); +} + +Key Database::get_schema_key(String const& schema_name) +{ + auto key = SchemaDef::make_key(); + key["schema_name"] = schema_name; + return key; +} + +RefPtr Database::get_schema(String const& schema_name) +{ + Key key = get_schema_key(schema_name); + auto schema_def_opt = m_schema_cache.get(key.hash()); + if (schema_def_opt.has_value()) + return schema_def_opt.value(); + auto schema_iterator = m_schemas->find(key); + if (schema_iterator.is_end() || (*schema_iterator != key)) { + warnln("Schema {} not found", schema_name); + return nullptr; + } + auto ret = SchemaDef::construct(*schema_iterator); + m_schema_cache.set(key.hash(), ret); + return ret; +} + +void Database::add_table(TableDef& table) +{ + m_tables->insert(table.key()); + for (auto& column : table.columns()) { + m_table_columns->insert(column.key()); + } +} + +Key Database::get_table_key(String const& schema_name, String const& table_name) +{ + auto key = TableDef::make_key(get_schema_key(schema_name)); + key["table_name"] = table_name; + return key; +} + +RefPtr Database::get_table(String const& schema, String const& name) +{ + Key key = get_table_key(schema, name); + auto table_def_opt = m_table_cache.get(key.hash()); + if (table_def_opt.has_value()) + return table_def_opt.value(); + auto table_iterator = m_tables->find(key); + if (table_iterator.is_end() || (*table_iterator != key)) { + warnln("Table {} not found", name); + return nullptr; + } + auto schema_def = get_schema(schema); + VERIFY(schema_def); + auto ret = TableDef::construct(schema_def, name); + ret->set_pointer((*table_iterator).pointer()); + m_table_cache.set(key.hash(), ret); + auto hash = ret->hash(); + auto column_key = ColumnDef::make_key(ret); + + for (auto column_iterator = m_table_columns->find(column_key); + !column_iterator.is_end() && ((*column_iterator)["table_hash"].to_u32().value() == hash); + column_iterator++) { + ret->append_column(*column_iterator); + } + return ret; +} + +Vector Database::select_all(TableDef const& table) +{ + VERIFY(m_table_cache.get(table.key().hash()).has_value()); + Vector ret; + for (auto pointer = table.pointer(); pointer; pointer = ret.last().next_pointer()) { + auto buffer_or_error = m_heap->read_block(pointer); + if (buffer_or_error.is_error()) + VERIFY_NOT_REACHED(); + ret.empend(table, pointer, buffer_or_error.value()); + } + return ret; +} + +Vector Database::match(TableDef const& table, Key const& key) +{ + VERIFY(m_table_cache.get(table.key().hash()).has_value()); + Vector ret; + + // TODO Match key against indexes defined on table. If found, + // use the index instead of scanning the table. + for (auto pointer = table.pointer(); pointer;) { + auto buffer_or_error = m_heap->read_block(pointer); + if (buffer_or_error.is_error()) + VERIFY_NOT_REACHED(); + Row row(table, pointer, buffer_or_error.value()); + if (row.match(key)) + ret.append(row); + pointer = ret.last().next_pointer(); + } + return ret; +} + +bool Database::insert(Row& row) +{ + VERIFY(m_table_cache.get(row.table()->key().hash()).has_value()); + row.set_pointer(m_heap->new_record_pointer()); + row.next_pointer(row.table()->pointer()); + update(row); + + // TODO update indexes defined on table. + + auto table_key = row.table()->key(); + table_key.set_pointer(row.pointer()); + VERIFY(m_tables->update_key_pointer(table_key)); + row.table()->set_pointer(row.pointer()); + return true; +} + +bool Database::update(Row& tuple) +{ + VERIFY(m_table_cache.get(tuple.table()->key().hash()).has_value()); + ByteBuffer buffer; + tuple.serialize(buffer); + m_heap->add_to_wal(tuple.pointer(), buffer); + + // TODO update indexes defined on table. + return true; +} + +} diff --git a/Userland/Libraries/LibSQL/Database.h b/Userland/Libraries/LibSQL/Database.h new file mode 100644 index 00000000000..5603dcc2e11 --- /dev/null +++ b/Userland/Libraries/LibSQL/Database.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, Jan de Visser + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace SQL { + +/** + * A Database object logically connects a Heap with the SQL data we want + * to store in it. It has BTree pointers for B-Trees holding the definitions + * of tables, columns, indexes, and other SQL objects. + */ +class Database : public Core::Object { + C_OBJECT(Database); + +public: + explicit Database(String); + ~Database() override = default; + + void commit() { m_heap->flush(); } + + void add_schema(SchemaDef const&); + static Key get_schema_key(String const&); + RefPtr get_schema(String const&); + + void add_table(TableDef& table); + static Key get_table_key(String const&, String const&); + RefPtr get_table(String const&, String const&); + + Vector select_all(TableDef const&); + Vector match(TableDef const&, Key const&); + bool insert(Row&); + bool update(Row&); + +private: + RefPtr m_heap; + RefPtr m_schemas; + RefPtr m_tables; + RefPtr m_table_columns; + + HashMap> m_schema_cache; + HashMap> m_table_cache; +}; + +} diff --git a/Userland/Libraries/LibSQL/Forward.h b/Userland/Libraries/LibSQL/Forward.h index 4d139afb8de..224404d1897 100644 --- a/Userland/Libraries/LibSQL/Forward.h +++ b/Userland/Libraries/LibSQL/Forward.h @@ -25,6 +25,7 @@ class ColumnNameExpression; class CommonTableExpression; class CommonTableExpressionList; class CreateTable; +class Database; class TupleDescriptor; struct TupleElement; class Delete; diff --git a/Userland/Libraries/LibSQL/Row.cpp b/Userland/Libraries/LibSQL/Row.cpp new file mode 100644 index 00000000000..9a5c94cc954 --- /dev/null +++ b/Userland/Libraries/LibSQL/Row.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021, Jan de Visser + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +namespace SQL { + +Row::Row() + : Tuple() +{ +} + +Row::Row(TupleDescriptor const& descriptor) + : Tuple(descriptor) +{ +} + +Row::Row(RefPtr table) + : Tuple(table->to_tuple_descriptor()) + , m_table(table) +{ +} + +Row::Row(RefPtr table, u32 pointer, ByteBuffer& buffer) + : Tuple(table->to_tuple_descriptor()) +{ + // FIXME Sanitize constructor situation in Tuple so this can be better + size_t offset = 0; + deserialize(buffer, offset); + deserialize_from(buffer, offset, m_next_pointer); + set_pointer(pointer); +} + +void Row::serialize(ByteBuffer& buffer) const +{ + Tuple::serialize(buffer); + serialize_to(buffer, next_pointer()); +} + +void Row::copy_from(Row const& other) +{ + Tuple::copy_from(other); + m_next_pointer = other.next_pointer(); +} + +} diff --git a/Userland/Libraries/LibSQL/Row.h b/Userland/Libraries/LibSQL/Row.h new file mode 100644 index 00000000000..27c98339f01 --- /dev/null +++ b/Userland/Libraries/LibSQL/Row.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, Jan de Visser + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +namespace SQL { + +/** + * A Tuple is an element of a sequential-access persistence data structure + * like a flat table. Like a key it has a definition for all its parts, + * but unlike a key this definition is not optional. + * + * FIXME Tuples should logically belong to a TupleStore object, but right now + * they stand by themselves; they contain a row's worth of data and a pointer + * to the next Tuple. + */ +class Row : public Tuple { +public: + Row(); + explicit Row(TupleDescriptor const&); + explicit Row(RefPtr); + Row(RefPtr, u32, ByteBuffer&); + Row(Row const&) = default; + virtual ~Row() override = default; + + [[nodiscard]] u32 next_pointer() const { return m_next_pointer; } + void next_pointer(u32 ptr) { m_next_pointer = ptr; } + RefPtr table() const { return m_table; } + virtual void serialize(ByteBuffer&) const override; + [[nodiscard]] virtual size_t data_length() const override { return Tuple::data_length() + sizeof(u32); } + +protected: + void copy_from(Row const&); + +private: + RefPtr m_table; + u32 m_next_pointer { 0 }; +}; + +}