diff --git a/Base/res/html/misc/cookie.html b/Base/res/html/misc/cookie.html index c3b8df92a99..70cbf8b96a2 100644 --- a/Base/res/html/misc/cookie.html +++ b/Base/res/html/misc/cookie.html @@ -6,6 +6,7 @@


+

Invalid cookies (the browser should reject these):

@@ -36,6 +37,10 @@ document.getElementById('cookies').innerHTML = document.cookie; } + function setPrettyLargeCookie() { + setCookie('cookie7=' + 'x'.repeat(2048)); + } + function setTooLargeCookie() { const cookie = 'name=' + 'x'.repeat(4 << 10); setCookie(cookie); diff --git a/Tests/LibSQL/TestSqlBtreeIndex.cpp b/Tests/LibSQL/TestSqlBtreeIndex.cpp index fe372c26ad9..081e2ca0689 100644 --- a/Tests/LibSQL/TestSqlBtreeIndex.cpp +++ b/Tests/LibSQL/TestSqlBtreeIndex.cpp @@ -131,7 +131,7 @@ NonnullRefPtr setup_btree(SQL::Serializer& serializer) auto root_pointer = serializer.heap().user_value(0); if (!root_pointer) { - root_pointer = serializer.heap().new_record_pointer(); + root_pointer = serializer.heap().request_new_block_index(); serializer.heap().set_user_value(0, root_pointer); } auto btree = SQL::BTree::construct(serializer, tuple_descriptor, true, root_pointer); diff --git a/Tests/LibSQL/TestSqlHashIndex.cpp b/Tests/LibSQL/TestSqlHashIndex.cpp index f648d8506dc..6a52cda14d0 100644 --- a/Tests/LibSQL/TestSqlHashIndex.cpp +++ b/Tests/LibSQL/TestSqlHashIndex.cpp @@ -129,7 +129,7 @@ NonnullRefPtr setup_hash_index(SQL::Serializer& serializer) auto directory_pointer = serializer.heap().user_value(0); if (!directory_pointer) { - directory_pointer = serializer.heap().new_record_pointer(); + directory_pointer = serializer.heap().request_new_block_index(); serializer.heap().set_user_value(0, directory_pointer); } auto hash_index = SQL::HashIndex::construct(serializer, tuple_descriptor, directory_pointer); diff --git a/Tests/LibSQL/TestSqlStatementExecution.cpp b/Tests/LibSQL/TestSqlStatementExecution.cpp index 62c5abc037d..b1691856f87 100644 --- a/Tests/LibSQL/TestSqlStatementExecution.cpp +++ b/Tests/LibSQL/TestSqlStatementExecution.cpp @@ -234,6 +234,24 @@ TEST_CASE(insert_with_placeholders) } } +TEST_CASE(insert_and_retrieve_long_text_value) +{ + ScopeGuard guard([]() { unlink(db_name); }); + auto database = SQL::Database::construct(db_name); + EXPECT(!database->open().is_error()); + create_table(database); + + StringBuilder sb; + MUST(sb.try_append_repeated('x', 8192)); + auto long_string = sb.string_view(); + auto result = execute(database, DeprecatedString::formatted("INSERT INTO TestSchema.TestTable VALUES ('{}', 0);", long_string)); + EXPECT(result.size() == 1); + + result = execute(database, "SELECT TextColumn FROM TestSchema.TestTable;"); + EXPECT_EQ(result.size(), 1u); + EXPECT_EQ(result[0].row[0], long_string); +} + TEST_CASE(select_from_empty_table) { ScopeGuard guard([]() { unlink(db_name); }); diff --git a/Userland/Libraries/LibSQL/BTree.cpp b/Userland/Libraries/LibSQL/BTree.cpp index 092262d9aa2..7ed5e57a036 100644 --- a/Userland/Libraries/LibSQL/BTree.cpp +++ b/Userland/Libraries/LibSQL/BTree.cpp @@ -37,13 +37,13 @@ void BTree::initialize_root() { if (pointer()) { if (serializer().has_block(pointer())) { - serializer().get_block(pointer()); + serializer().read_storage(pointer()); m_root = serializer().make_and_deserialize(*this, pointer()); } else { m_root = make(*this, nullptr, pointer()); } } else { - set_pointer(new_record_pointer()); + set_pointer(request_new_block_index()); m_root = make(*this, nullptr, pointer()); if (on_new_root) on_new_root(); @@ -53,7 +53,7 @@ void BTree::initialize_root() TreeNode* BTree::new_root() { - set_pointer(new_record_pointer()); + set_pointer(request_new_block_index()); m_root = make(*this, nullptr, m_root.leak_ptr(), pointer()); serializer().serialize_and_write(*m_root.ptr()); if (on_new_root) diff --git a/Userland/Libraries/LibSQL/Database.cpp b/Userland/Libraries/LibSQL/Database.cpp index 33e6b17d0e7..9c2921c013e 100644 --- a/Userland/Libraries/LibSQL/Database.cpp +++ b/Userland/Libraries/LibSQL/Database.cpp @@ -197,9 +197,9 @@ ErrorOr> Database::match(TableDef& table, Key const& key) ErrorOr Database::insert(Row& row) { VERIFY(m_table_cache.get(row.table().key().hash()).has_value()); - // TODO Check constraints + // TODO: implement table constraints such as unique, foreign key, etc. - row.set_pointer(m_heap->new_record_pointer()); + row.set_pointer(m_heap->request_new_block_index()); row.set_next_pointer(row.table().pointer()); TRY(update(row)); @@ -244,7 +244,8 @@ ErrorOr Database::remove(Row& row) ErrorOr Database::update(Row& tuple) { VERIFY(m_table_cache.get(tuple.table().key().hash()).has_value()); - // TODO Check constraints + // TODO: implement table constraints such as unique, foreign key, etc. + m_serializer.reset(); m_serializer.serialize_and_write(tuple); diff --git a/Userland/Libraries/LibSQL/HashIndex.cpp b/Userland/Libraries/LibSQL/HashIndex.cpp index f9998cd0551..9c24ee6ca31 100644 --- a/Userland/Libraries/LibSQL/HashIndex.cpp +++ b/Userland/Libraries/LibSQL/HashIndex.cpp @@ -132,7 +132,7 @@ bool HashBucket::insert(Key const& key) m_hash_index.serializer().deserialize_block_to(pointer(), *this); if (find_key_in_bucket(key).has_value()) return false; - if ((length() + key.length()) > Heap::BLOCK_SIZE) { + if (length() + key.length() > Block::DATA_SIZE) { dbgln_if(SQL_DEBUG, "Adding key {} would make length exceed block size", key.to_deprecated_string()); return false; } @@ -205,7 +205,7 @@ HashIndex::HashIndex(Serializer& serializer, NonnullRefPtr cons , m_buckets() { if (!first_node) - set_pointer(new_record_pointer()); + set_pointer(request_new_block_index()); if (serializer.has_block(first_node)) { u32 pointer = first_node; do { @@ -216,14 +216,14 @@ HashIndex::HashIndex(Serializer& serializer, NonnullRefPtr cons pointer = m_nodes.last(); // FIXME Ugly } while (pointer); } else { - auto bucket = append_bucket(0u, 1u, new_record_pointer()); + auto bucket = append_bucket(0u, 1u, request_new_block_index()); bucket->m_inflated = true; serializer.serialize_and_write(*bucket); - bucket = append_bucket(1u, 1u, new_record_pointer()); + bucket = append_bucket(1u, 1u, request_new_block_index()); bucket->m_inflated = true; serializer.serialize_and_write(*bucket); m_nodes.append(first_node); - write_directory_to_write_ahead_log(); + write_directory(); } } @@ -247,7 +247,7 @@ HashBucket* HashIndex::get_bucket_for_insert(Key const& key) do { dbgln_if(SQL_DEBUG, "HashIndex::get_bucket_for_insert({}) bucket {} of {}", key.to_deprecated_string(), key_hash % size(), size()); auto bucket = get_bucket(key_hash % size()); - if (bucket->length() + key.length() < Heap::BLOCK_SIZE) + if (bucket->length() + key.length() < Block::DATA_SIZE) return bucket; dbgln_if(SQL_DEBUG, "Bucket is full (bucket size {}/length {} key length {}). Expanding directory", bucket->size(), bucket->length(), key.length()); @@ -266,7 +266,7 @@ HashBucket* HashIndex::get_bucket_for_insert(Key const& key) for (auto entry_index = (int)bucket->m_entries.size() - 1; entry_index >= 0; entry_index--) { if (bucket->m_entries[entry_index].hash() % size() == ix) { if (!sub_bucket->pointer()) - sub_bucket->set_pointer(new_record_pointer()); + sub_bucket->set_pointer(request_new_block_index()); sub_bucket->insert(bucket->m_entries.take(entry_index)); moved++; } @@ -283,10 +283,10 @@ HashBucket* HashIndex::get_bucket_for_insert(Key const& key) dbgln_if(SQL_DEBUG, "Nothing redistributed from bucket #{}", base_index); bucket->set_local_depth(bucket->local_depth() + 1); serializer().serialize_and_write(*bucket); - write_directory_to_write_ahead_log(); + write_directory(); auto bucket_after_redistribution = get_bucket(key_hash % size()); - if (bucket_after_redistribution->length() + key.length() < Heap::BLOCK_SIZE) + if (bucket_after_redistribution->length() + key.length() < Block::DATA_SIZE) return bucket_after_redistribution; } expand(); @@ -304,14 +304,14 @@ void HashIndex::expand() bucket->m_inflated = true; } m_global_depth++; - write_directory_to_write_ahead_log(); + write_directory(); } -void HashIndex::write_directory_to_write_ahead_log() +void HashIndex::write_directory() { auto num_nodes_required = (size() / HashDirectoryNode::max_pointers_in_node()) + 1; while (m_nodes.size() < num_nodes_required) - m_nodes.append(new_record_pointer()); + m_nodes.append(request_new_block_index()); size_t offset = 0u; size_t num_node = 0u; diff --git a/Userland/Libraries/LibSQL/HashIndex.h b/Userland/Libraries/LibSQL/HashIndex.h index e320a6f5d92..a46a7fd3767 100644 --- a/Userland/Libraries/LibSQL/HashIndex.h +++ b/Userland/Libraries/LibSQL/HashIndex.h @@ -82,7 +82,7 @@ private: HashIndex(Serializer&, NonnullRefPtr const&, u32); void expand(); - void write_directory_to_write_ahead_log(); + void write_directory(); HashBucket* append_bucket(u32 index, u32 local_depth, u32 pointer); HashBucket* get_bucket_for_insert(Key const&); [[nodiscard]] HashBucket* get_bucket_by_index(u32 index); @@ -104,7 +104,7 @@ public: void serialize(Serializer&) const; [[nodiscard]] u32 number_of_pointers() const { return min(max_pointers_in_node(), m_hash_index.size() - m_offset); } [[nodiscard]] bool is_last() const { return m_is_last; } - static constexpr size_t max_pointers_in_node() { return (Heap::BLOCK_SIZE - 3 * sizeof(u32)) / (2 * sizeof(u32)); } + static constexpr size_t max_pointers_in_node() { return (Block::DATA_SIZE - 3 * sizeof(u32)) / (2 * sizeof(u32)); } private: HashIndex& m_hash_index; diff --git a/Userland/Libraries/LibSQL/Heap.cpp b/Userland/Libraries/LibSQL/Heap.cpp index 10e79d3f5f9..d3389fbf2fd 100644 --- a/Userland/Libraries/LibSQL/Heap.cpp +++ b/Userland/Libraries/LibSQL/Heap.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Jan de Visser + * Copyright (c) 2023, Jelle Raaijmakers * * SPDX-License-Identifier: BSD-2-Clause */ @@ -42,8 +43,10 @@ ErrorOr Heap::open() } else { file_size = stat_buffer.st_size; } - if (file_size > 0) - m_next_block = m_end_of_file = file_size / BLOCK_SIZE; + if (file_size > 0) { + m_next_block = file_size / Block::SIZE; + m_highest_block_written = m_next_block - 1; + } auto file = TRY(Core::File::open(name(), Core::File::OpenMode::ReadWrite)); m_file = TRY(Core::BufferedFile::create(move(file))); @@ -54,7 +57,7 @@ ErrorOr Heap::open() return error_maybe.release_error(); } } else { - initialize_zero_block(); + TRY(initialize_zero_block()); } // FIXME: We should more gracefully handle version incompatibilities. For now, we drop the database. @@ -66,118 +69,137 @@ ErrorOr Heap::open() return open(); } - dbgln_if(SQL_DEBUG, "Heap file {} opened. Size = {}", name(), size()); + dbgln_if(SQL_DEBUG, "Heap file {} opened; number of blocks = {}", name(), m_highest_block_written); return {}; } -ErrorOr Heap::read_block(u32 block) +bool Heap::has_block(Block::Index index) const { - if (!m_file) { - warnln("Heap({})::read_block({}): Heap file not opened"sv, name(), block); - return Error::from_string_literal("Heap()::read_block(): Heap file not opened"); + return index <= m_highest_block_written || m_write_ahead_log.contains(index); +} + +ErrorOr Heap::read_storage(Block::Index index) +{ + dbgln_if(SQL_DEBUG, "{}({})", __FUNCTION__, index); + + // Reconstruct the data storage from a potential chain of blocks + ByteBuffer data; + while (index > 0) { + auto block = TRY(read_block(index)); + dbgln_if(SQL_DEBUG, " -> {} bytes", block.size_in_bytes()); + TRY(data.try_append(block.data().bytes().slice(0, block.size_in_bytes()))); + index = block.next_block(); } + return data; +} - if (auto buffer = m_write_ahead_log.get(block); buffer.has_value()) - return TRY(ByteBuffer::copy(*buffer)); +ErrorOr Heap::write_storage(Block::Index index, ReadonlyBytes data) +{ + dbgln_if(SQL_DEBUG, "{}({}, {} bytes)", __FUNCTION__, index, data.size()); + VERIFY(data.size() > 0); - if (block >= m_next_block) { - warnln("Heap({})::read_block({}): block # out of range (>= {})"sv, name(), block, m_next_block); - return Error::from_string_literal("Heap()::read_block(): block # out of range"); + // Split up the storage across multiple blocks if necessary, creating a chain + u32 remaining_size = static_cast(data.size()); + u32 offset_in_data = 0; + while (remaining_size > 0) { + auto block_data_size = AK::min(remaining_size, Block::DATA_SIZE); + remaining_size -= block_data_size; + auto next_block_index = (remaining_size > 0) ? request_new_block_index() : 0; + + auto block_data = TRY(ByteBuffer::create_uninitialized(block_data_size)); + block_data.bytes().overwrite(0, data.offset(offset_in_data), block_data_size); + + TRY(write_block({ index, block_data_size, next_block_index, move(block_data) })); + + index = next_block_index; + offset_in_data += block_data_size; } + return {}; +} - dbgln_if(SQL_DEBUG, "Read heap block {}", block); - TRY(seek_block(block)); +ErrorOr Heap::read_raw_block(Block::Index index) +{ + VERIFY(m_file); + VERIFY(index < m_next_block); - auto buffer = TRY(ByteBuffer::create_uninitialized(BLOCK_SIZE)); + if (auto data = m_write_ahead_log.get(index); data.has_value()) + return data.value(); + + TRY(m_file->seek(index * Block::SIZE, SeekMode::SetPosition)); + auto buffer = TRY(ByteBuffer::create_uninitialized(Block::SIZE)); TRY(m_file->read_until_filled(buffer)); - - dbgln_if(SQL_DEBUG, "{:hex-dump}", buffer.bytes().trim(8)); - return buffer; } -ErrorOr Heap::write_block(u32 block, ByteBuffer& buffer) +ErrorOr Heap::read_block(Block::Index index) { - if (!m_file) { - warnln("Heap({})::write_block({}): Heap file not opened"sv, name(), block); - return Error::from_string_literal("Heap()::write_block(): Heap file not opened"); - } - if (block > m_next_block) { - warnln("Heap({})::write_block({}): block # out of range (> {})"sv, name(), block, m_next_block); - return Error::from_string_literal("Heap()::write_block(): block # out of range"); - } - if (buffer.size() > BLOCK_SIZE) { - warnln("Heap({})::write_block({}): Oversized block ({} > {})"sv, name(), block, buffer.size(), BLOCK_SIZE); - return Error::from_string_literal("Heap()::write_block(): Oversized block"); - } + dbgln_if(SQL_DEBUG, "Read heap block {}", index); - dbgln_if(SQL_DEBUG, "Write heap block {} size {}", block, buffer.size()); - TRY(seek_block(block)); + auto buffer = TRY(read_raw_block(index)); + auto size_in_bytes = *reinterpret_cast(buffer.offset_pointer(0)); + auto next_block = *reinterpret_cast(buffer.offset_pointer(sizeof(u32))); + auto data = TRY(buffer.slice(Block::HEADER_SIZE, Block::DATA_SIZE)); - if (auto current_size = buffer.size(); current_size < BLOCK_SIZE) { - TRY(buffer.try_resize(BLOCK_SIZE)); - memset(buffer.offset_pointer(current_size), 0, BLOCK_SIZE - current_size); - } - - dbgln_if(SQL_DEBUG, "{:hex-dump}", buffer.bytes().trim(8)); - TRY(m_file->write_until_depleted(buffer)); - - if (block == m_end_of_file) - m_end_of_file++; - return {}; + return Block { index, size_in_bytes, next_block, move(data) }; } -ErrorOr Heap::seek_block(u32 block) +ErrorOr Heap::write_raw_block(Block::Index index, ReadonlyBytes data) { - if (!m_file) { - warnln("Heap({})::seek_block({}): Heap file not opened"sv, name(), block); - return Error::from_string_literal("Heap()::seek_block(): Heap file not opened"); - } - if (block > m_end_of_file) { - warnln("Heap({})::seek_block({}): Cannot seek beyond end of file at block {}"sv, name(), block, m_end_of_file); - return Error::from_string_literal("Heap()::seek_block(): Cannot seek beyond end of file"); - } + dbgln_if(SQL_DEBUG, "Write raw block {}", index); - if (block == m_end_of_file) - TRY(m_file->seek(0, SeekMode::FromEndPosition)); - else - TRY(m_file->seek(block * BLOCK_SIZE, SeekMode::SetPosition)); - - return {}; -} - -u32 Heap::new_record_pointer() -{ VERIFY(m_file); - if (m_free_list) { - auto block_or_error = read_block(m_free_list); - if (block_or_error.is_error()) { - warnln("FREE LIST CORRUPTION"); - VERIFY_NOT_REACHED(); - } - auto new_pointer = m_free_list; - memcpy(&m_free_list, block_or_error.value().offset_pointer(0), sizeof(u32)); - update_zero_block(); - return new_pointer; - } - return m_next_block++; + VERIFY(data.size() == Block::SIZE); + + TRY(m_file->seek(index * Block::SIZE, SeekMode::SetPosition)); + TRY(m_file->write_until_depleted(data)); + + if (index > m_highest_block_written) + m_highest_block_written = index; + + return {}; +} + +ErrorOr Heap::write_raw_block_to_wal(Block::Index index, ByteBuffer&& data) +{ + dbgln_if(SQL_DEBUG, "{}(): adding raw block {} to WAL", __FUNCTION__, index); + VERIFY(index < m_next_block); + VERIFY(data.size() == Block::SIZE); + + TRY(m_write_ahead_log.try_set(index, move(data))); + + return {}; +} + +ErrorOr Heap::write_block(Block const& block) +{ + VERIFY(block.index() < m_next_block); + VERIFY(block.next_block() < m_next_block); + VERIFY(block.data().size() <= Block::DATA_SIZE); + + auto size_in_bytes = block.size_in_bytes(); + auto next_block = block.next_block(); + + auto heap_data = TRY(ByteBuffer::create_zeroed(Block::SIZE)); + heap_data.overwrite(0, &size_in_bytes, sizeof(size_in_bytes)); + heap_data.overwrite(sizeof(size_in_bytes), &next_block, sizeof(next_block)); + + block.data().bytes().copy_to(heap_data.bytes().slice(Block::HEADER_SIZE)); + + return write_raw_block_to_wal(block.index(), move(heap_data)); } ErrorOr Heap::flush() { VERIFY(m_file); - Vector blocks; - for (auto& wal_entry : m_write_ahead_log) - blocks.append(wal_entry.key); - quick_sort(blocks); - for (auto& block : blocks) { - auto buffer_it = m_write_ahead_log.find(block); - VERIFY(buffer_it != m_write_ahead_log.end()); - dbgln_if(SQL_DEBUG, "Flushing block {} to {}", block, name()); - TRY(write_block(block, buffer_it->value)); + auto indices = m_write_ahead_log.keys(); + quick_sort(indices); + for (auto index : indices) { + dbgln_if(SQL_DEBUG, "Flushing block {} to {}", index, name()); + auto& data = m_write_ahead_log.get(index).value(); + TRY(write_raw_block(index, data)); } m_write_ahead_log.clear(); - dbgln_if(SQL_DEBUG, "WAL flushed. Heap size = {}", size()); + dbgln_if(SQL_DEBUG, "WAL flushed; new number of blocks = {}", m_highest_block_written); return {}; } @@ -186,37 +208,33 @@ constexpr static auto VERSION_OFFSET = FILE_ID.length(); constexpr static auto SCHEMAS_ROOT_OFFSET = VERSION_OFFSET + sizeof(u32); constexpr static auto TABLES_ROOT_OFFSET = SCHEMAS_ROOT_OFFSET + sizeof(u32); constexpr static auto TABLE_COLUMNS_ROOT_OFFSET = TABLES_ROOT_OFFSET + sizeof(u32); -constexpr static auto FREE_LIST_OFFSET = TABLE_COLUMNS_ROOT_OFFSET + sizeof(u32); -constexpr static auto USER_VALUES_OFFSET = FREE_LIST_OFFSET + sizeof(u32); +constexpr static auto USER_VALUES_OFFSET = TABLE_COLUMNS_ROOT_OFFSET + sizeof(u32); ErrorOr Heap::read_zero_block() { - auto buffer = TRY(read_block(0)); - auto file_id_buffer = TRY(buffer.slice(0, FILE_ID.length())); + dbgln_if(SQL_DEBUG, "Read zero block from {}", name()); + + auto block = TRY(read_raw_block(0)); + auto file_id_buffer = TRY(block.slice(0, FILE_ID.length())); auto file_id = StringView(file_id_buffer); if (file_id != FILE_ID) { warnln("{}: Zero page corrupt. This is probably not a {} heap file"sv, name(), FILE_ID); return Error::from_string_literal("Heap()::read_zero_block(): Zero page corrupt. This is probably not a SerenitySQL heap file"); } - dbgln_if(SQL_DEBUG, "Read zero block from {}", name()); - - memcpy(&m_version, buffer.offset_pointer(VERSION_OFFSET), sizeof(u32)); + memcpy(&m_version, block.offset_pointer(VERSION_OFFSET), sizeof(u32)); dbgln_if(SQL_DEBUG, "Version: {}.{}", (m_version & 0xFFFF0000) >> 16, (m_version & 0x0000FFFF)); - memcpy(&m_schemas_root, buffer.offset_pointer(SCHEMAS_ROOT_OFFSET), sizeof(u32)); + memcpy(&m_schemas_root, block.offset_pointer(SCHEMAS_ROOT_OFFSET), sizeof(u32)); dbgln_if(SQL_DEBUG, "Schemas root node: {}", m_schemas_root); - memcpy(&m_tables_root, buffer.offset_pointer(TABLES_ROOT_OFFSET), sizeof(u32)); + memcpy(&m_tables_root, block.offset_pointer(TABLES_ROOT_OFFSET), sizeof(u32)); dbgln_if(SQL_DEBUG, "Tables root node: {}", m_tables_root); - memcpy(&m_table_columns_root, buffer.offset_pointer(TABLE_COLUMNS_ROOT_OFFSET), sizeof(u32)); + memcpy(&m_table_columns_root, block.offset_pointer(TABLE_COLUMNS_ROOT_OFFSET), sizeof(u32)); dbgln_if(SQL_DEBUG, "Table columns root node: {}", m_table_columns_root); - memcpy(&m_free_list, buffer.offset_pointer(FREE_LIST_OFFSET), sizeof(u32)); - dbgln_if(SQL_DEBUG, "Free list: {}", m_free_list); - - memcpy(m_user_values.data(), buffer.offset_pointer(USER_VALUES_OFFSET), m_user_values.size() * sizeof(u32)); + memcpy(m_user_values.data(), block.offset_pointer(USER_VALUES_OFFSET), m_user_values.size() * sizeof(u32)); for (auto ix = 0u; ix < m_user_values.size(); ix++) { if (m_user_values[ix]) dbgln_if(SQL_DEBUG, "User value {}: {}", ix, m_user_values[ix]); @@ -224,43 +242,40 @@ ErrorOr Heap::read_zero_block() return {}; } -void Heap::update_zero_block() +ErrorOr Heap::update_zero_block() { dbgln_if(SQL_DEBUG, "Write zero block to {}", name()); dbgln_if(SQL_DEBUG, "Version: {}.{}", (m_version & 0xFFFF0000) >> 16, (m_version & 0x0000FFFF)); dbgln_if(SQL_DEBUG, "Schemas root node: {}", m_schemas_root); dbgln_if(SQL_DEBUG, "Tables root node: {}", m_tables_root); dbgln_if(SQL_DEBUG, "Table Columns root node: {}", m_table_columns_root); - dbgln_if(SQL_DEBUG, "Free list: {}", m_free_list); for (auto ix = 0u; ix < m_user_values.size(); ix++) { - if (m_user_values[ix]) + if (m_user_values[ix] > 0) dbgln_if(SQL_DEBUG, "User value {}: {}", ix, m_user_values[ix]); } - // FIXME: Handle an OOM failure here. - auto buffer = ByteBuffer::create_zeroed(BLOCK_SIZE).release_value_but_fixme_should_propagate_errors(); - buffer.overwrite(0, FILE_ID.characters_without_null_termination(), FILE_ID.length()); - buffer.overwrite(VERSION_OFFSET, &m_version, sizeof(u32)); - buffer.overwrite(SCHEMAS_ROOT_OFFSET, &m_schemas_root, sizeof(u32)); - buffer.overwrite(TABLES_ROOT_OFFSET, &m_tables_root, sizeof(u32)); - buffer.overwrite(TABLE_COLUMNS_ROOT_OFFSET, &m_table_columns_root, sizeof(u32)); - buffer.overwrite(FREE_LIST_OFFSET, &m_free_list, sizeof(u32)); - buffer.overwrite(USER_VALUES_OFFSET, m_user_values.data(), m_user_values.size() * sizeof(u32)); + auto buffer = TRY(ByteBuffer::create_zeroed(Block::SIZE)); + auto buffer_bytes = buffer.bytes(); + buffer_bytes.overwrite(0, FILE_ID.characters_without_null_termination(), FILE_ID.length()); + buffer_bytes.overwrite(VERSION_OFFSET, &m_version, sizeof(u32)); + buffer_bytes.overwrite(SCHEMAS_ROOT_OFFSET, &m_schemas_root, sizeof(u32)); + buffer_bytes.overwrite(TABLES_ROOT_OFFSET, &m_tables_root, sizeof(u32)); + buffer_bytes.overwrite(TABLE_COLUMNS_ROOT_OFFSET, &m_table_columns_root, sizeof(u32)); + buffer_bytes.overwrite(USER_VALUES_OFFSET, m_user_values.data(), m_user_values.size() * sizeof(u32)); - add_to_wal(0, buffer); + return write_raw_block_to_wal(0, move(buffer)); } -void Heap::initialize_zero_block() +ErrorOr Heap::initialize_zero_block() { m_version = VERSION; m_schemas_root = 0; m_tables_root = 0; m_table_columns_root = 0; m_next_block = 1; - m_free_list = 0; for (auto& user : m_user_values) user = 0u; - update_zero_block(); + return update_zero_block(); } } diff --git a/Userland/Libraries/LibSQL/Heap.h b/Userland/Libraries/LibSQL/Heap.h index 00ca39f7a9e..62380ae2f25 100644 --- a/Userland/Libraries/LibSQL/Heap.h +++ b/Userland/Libraries/LibSQL/Heap.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Jan de Visser + * Copyright (c) 2023, Jelle Raaijmakers * * SPDX-License-Identifier: BSD-2-Clause */ @@ -15,6 +16,43 @@ namespace SQL { +/** + * A Block represents a single discrete chunk of 1024 bytes inside the Heap, and + * acts as the container format for the actual data we are storing. This structure + * is used for everything except block 0, the zero / super block. + * + * If data needs to be stored that is larger than 1016 bytes, Blocks are chained + * together by setting the next block index and the data is reconstructed by + * repeatedly reading blocks until the next block index is 0. + */ +class Block { +public: + typedef u32 Index; + + static constexpr u32 SIZE = 1024; + static constexpr u32 HEADER_SIZE = sizeof(u32) + sizeof(Index); + static constexpr u32 DATA_SIZE = SIZE - HEADER_SIZE; + + Block(Index index, u32 size_in_bytes, Index next_block, ByteBuffer data) + : m_index(index) + , m_size_in_bytes(size_in_bytes) + , m_next_block(next_block) + , m_data(move(data)) + { + } + + Index index() const { return m_index; } + u32 size_in_bytes() const { return m_size_in_bytes; } + Index next_block() const { return m_next_block; } + ByteBuffer const& data() const { return m_data; } + +private: + Index m_index; + u32 m_size_in_bytes; + Index m_next_block; + ByteBuffer m_data; +}; + /** * A Heap is a logical container for database (SQL) data. Conceptually a * Heap can be a database file, or a memory block, or another storage medium. @@ -23,30 +61,25 @@ namespace SQL { * * A Heap can be thought of the backing storage of a single database. It's * assumed that a single SQL database is backed by a single Heap. - * - * Currently only B-Trees and tuple stores are implemented. */ class Heap : public Core::Object { C_OBJECT(Heap); public: - static constexpr u32 VERSION = 3; - static constexpr u32 BLOCK_SIZE = 1024; + static constexpr u32 VERSION = 4; virtual ~Heap() override; ErrorOr open(); - u32 size() const { return m_end_of_file; } - ErrorOr read_block(u32); - [[nodiscard]] u32 new_record_pointer(); - [[nodiscard]] bool valid() const { return static_cast(m_file); } + bool has_block(Block::Index) const; + [[nodiscard]] Block::Index request_new_block_index() { return m_next_block++; } u32 schemas_root() const { return m_schemas_root; } void set_schemas_root(u32 root) { m_schemas_root = root; - update_zero_block(); + update_zero_block().release_value_but_fixme_should_propagate_errors(); } u32 tables_root() const { return m_tables_root; } @@ -54,7 +87,7 @@ public: void set_tables_root(u32 root) { m_tables_root = root; - update_zero_block(); + update_zero_block().release_value_but_fixme_should_propagate_errors(); } u32 table_columns_root() const { return m_table_columns_root; } @@ -62,7 +95,7 @@ public: void set_table_columns_root(u32 root) { m_table_columns_root = root; - update_zero_block(); + update_zero_block().release_value_but_fixme_should_propagate_errors(); } u32 version() const { return m_version; } @@ -74,31 +107,30 @@ public: void set_user_value(size_t index, u32 value) { m_user_values[index] = value; - update_zero_block(); + update_zero_block().release_value_but_fixme_should_propagate_errors(); } - void add_to_wal(u32 block, ByteBuffer& buffer) - { - dbgln_if(SQL_DEBUG, "Adding to WAL: block #{}, size {}", block, buffer.size()); - dbgln_if(SQL_DEBUG, "{:hex-dump}", buffer.bytes().trim(8)); - m_write_ahead_log.set(block, buffer); - } + ErrorOr read_storage(Block::Index); + ErrorOr write_storage(Block::Index, ReadonlyBytes); ErrorOr flush(); private: explicit Heap(DeprecatedString); - ErrorOr write_block(u32, ByteBuffer&); - ErrorOr seek_block(u32); + ErrorOr read_raw_block(Block::Index); + ErrorOr write_raw_block(Block::Index, ReadonlyBytes); + ErrorOr write_raw_block_to_wal(Block::Index, ByteBuffer&&); + + ErrorOr read_block(Block::Index); + ErrorOr write_block(Block const&); ErrorOr read_zero_block(); - void initialize_zero_block(); - void update_zero_block(); + ErrorOr initialize_zero_block(); + ErrorOr update_zero_block(); OwnPtr m_file; - u32 m_free_list { 0 }; + Block::Index m_highest_block_written { 0 }; u32 m_next_block { 1 }; - u32 m_end_of_file { 1 }; u32 m_schemas_root { 0 }; u32 m_tables_root { 0 }; u32 m_table_columns_root { 0 }; diff --git a/Userland/Libraries/LibSQL/Index.h b/Userland/Libraries/LibSQL/Index.h index 6f0209ffa97..be8de935903 100644 --- a/Userland/Libraries/LibSQL/Index.h +++ b/Userland/Libraries/LibSQL/Index.h @@ -48,8 +48,7 @@ protected: [[nodiscard]] Serializer& serializer() { return m_serializer; } void set_pointer(u32 pointer) { m_pointer = pointer; } - u32 new_record_pointer() { return m_serializer.new_record_pointer(); } - // ByteBuffer read_block(u32); + u32 request_new_block_index() { return m_serializer.request_new_block_index(); } private: Serializer m_serializer; diff --git a/Userland/Libraries/LibSQL/Serializer.h b/Userland/Libraries/LibSQL/Serializer.h index 8a2be68e3d2..53063731bd6 100644 --- a/Userland/Libraries/LibSQL/Serializer.h +++ b/Userland/Libraries/LibSQL/Serializer.h @@ -12,7 +12,6 @@ #include #include #include -#include namespace SQL { @@ -25,12 +24,9 @@ public: { } - void get_block(u32 pointer) + void read_storage(Block::Index block_index) { - auto buffer_or_error = m_heap->read_block(pointer); - if (buffer_or_error.is_error()) - VERIFY_NOT_REACHED(); - m_buffer = buffer_or_error.value(); + m_buffer = m_heap->read_storage(block_index).release_value_but_fixme_should_propagate_errors(); m_current_offset = 0; } @@ -48,14 +44,14 @@ public: template T deserialize_block(u32 pointer, Args&&... args) { - get_block(pointer); + read_storage(pointer); return deserialize(forward(args)...); } template void deserialize_block_to(u32 pointer, T& t) { - get_block(pointer); + read_storage(pointer); return deserialize_to(t); } @@ -111,19 +107,19 @@ public: VERIFY(!m_heap.is_null()); reset(); serialize(t); - m_heap->add_to_wal(t.pointer(), m_buffer); + m_heap->write_storage(t.pointer(), m_buffer).release_value_but_fixme_should_propagate_errors(); return true; } [[nodiscard]] size_t offset() const { return m_current_offset; } - u32 new_record_pointer() + u32 request_new_block_index() { - return m_heap->new_record_pointer(); + return m_heap->request_new_block_index(); } bool has_block(u32 pointer) const { - return pointer < m_heap->size(); + return m_heap->has_block(pointer); } Heap& heap() diff --git a/Userland/Libraries/LibSQL/TreeNode.cpp b/Userland/Libraries/LibSQL/TreeNode.cpp index d25676203c9..f9972cf382b 100644 --- a/Userland/Libraries/LibSQL/TreeNode.cpp +++ b/Userland/Libraries/LibSQL/TreeNode.cpp @@ -51,7 +51,7 @@ void DownPointer::deserialize(Serializer& serializer) { if (m_node || !m_pointer) return; - serializer.get_block(m_pointer); + serializer.read_storage(m_pointer); m_node = serializer.make_and_deserialize(m_owner->tree(), m_owner, m_pointer); } @@ -87,7 +87,7 @@ TreeNode::TreeNode(BTree& tree, TreeNode* up, DownPointer& left, u32 pointer) m_down.append(DownPointer(this, left)); m_is_leaf = left.pointer() == 0; if (!pointer) - set_pointer(m_tree.new_record_pointer()); + set_pointer(m_tree.request_new_block_index()); } TreeNode::TreeNode(BTree& tree, TreeNode* up, TreeNode* left, u32 pointer) @@ -271,7 +271,7 @@ void TreeNode::just_insert(Key const& key, TreeNode* right) m_entries.insert(ix, key); VERIFY(is_leaf() == (right == nullptr)); m_down.insert(ix + 1, DownPointer(this, right)); - if (length() > Heap::BLOCK_SIZE) { + if (length() > Block::DATA_SIZE) { split(); } else { dump_if(SQL_DEBUG, "To WAL"); @@ -283,7 +283,7 @@ void TreeNode::just_insert(Key const& key, TreeNode* right) m_entries.append(key); m_down.empend(this, right); - if (length() > Heap::BLOCK_SIZE) { + if (length() > Block::DATA_SIZE) { split(); } else { dump_if(SQL_DEBUG, "To WAL");