diff --git a/Tests/LibCompress/TestDeflate.cpp b/Tests/LibCompress/TestDeflate.cpp index dc713431de3..8c96024affc 100644 --- a/Tests/LibCompress/TestDeflate.cpp +++ b/Tests/LibCompress/TestDeflate.cpp @@ -118,7 +118,7 @@ TEST_CASE(deflate_round_trip_store) auto original = ByteBuffer::create_uninitialized(1024).release_value(); fill_with_random(original.data(), 1024); auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::STORE); - EXPECT(compressed.has_value()); + EXPECT(!compressed.is_error()); auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value()); EXPECT(!uncompressed.is_error()); EXPECT(uncompressed.value() == original); @@ -130,7 +130,7 @@ TEST_CASE(deflate_round_trip_compress) fill_with_random(original.data(), 1024); // we pre-filled the second half with 0s to make sure we test back references as well // Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST); - EXPECT(compressed.has_value()); + EXPECT(!compressed.is_error()); auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value()); EXPECT(!uncompressed.is_error()); EXPECT(uncompressed.value() == original); @@ -143,7 +143,7 @@ TEST_CASE(deflate_round_trip_compress_large) fill_with_random(original.data(), size); // Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST); - EXPECT(compressed.has_value()); + EXPECT(!compressed.is_error()); auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value()); EXPECT(!uncompressed.is_error()); EXPECT(uncompressed.value() == original); @@ -154,5 +154,5 @@ TEST_CASE(deflate_compress_literals) // This byte array is known to not produce any back references with our lz77 implementation even at the highest compression settings Array test { 0, 0, 0, 0, 0x72, 0, 0, 0xee, 0, 0, 0, 0x26, 0, 0, 0, 0x28, 0, 0, 0x72 }; auto compressed = Compress::DeflateCompressor::compress_all(test, Compress::DeflateCompressor::CompressionLevel::GOOD); - EXPECT(compressed.has_value()); + EXPECT(!compressed.is_error()); } diff --git a/Userland/Libraries/LibCompress/Deflate.cpp b/Userland/Libraries/LibCompress/Deflate.cpp index 3b704340cb3..cc0f7869133 100644 --- a/Userland/Libraries/LibCompress/Deflate.cpp +++ b/Userland/Libraries/LibCompress/Deflate.cpp @@ -116,9 +116,10 @@ ErrorOr CanonicalCode::read_symbol(Core::Stream::LittleEndianInputBitStream } } -void CanonicalCode::write_symbol(OutputBitStream& stream, u32 symbol) const +ErrorOr CanonicalCode::write_symbol(Core::Stream::LittleEndianOutputBitStream& stream, u32 symbol) const { - stream.write_bits(m_bit_codes[symbol], m_bit_code_lengths[symbol]); + TRY(stream.write_bits(m_bit_codes[symbol], m_bit_code_lengths[symbol])); + return {}; } DeflateDecompressor::CompressedBlock::CompressedBlock(DeflateDecompressor& decompressor, CanonicalCode literal_codes, Optional distance_codes) @@ -437,10 +438,10 @@ ErrorOr DeflateDecompressor::decode_codes(CanonicalCode& literal_code, Opt return {}; } -DeflateCompressor::DeflateCompressor(OutputStream& stream, CompressionLevel compression_level) +DeflateCompressor::DeflateCompressor(Core::Stream::Handle stream, CompressionLevel compression_level) : m_compression_level(compression_level) , m_compression_constants(compression_constants[static_cast(m_compression_level)]) - , m_output_stream(stream) + , m_output_stream(Core::Stream::LittleEndianOutputBitStream::construct(move(stream)).release_value_but_fixme_should_propagate_errors()) { m_symbol_frequencies.fill(0); m_distance_frequencies.fill(0); @@ -451,7 +452,12 @@ DeflateCompressor::~DeflateCompressor() VERIFY(m_finished); } -size_t DeflateCompressor::write(ReadonlyBytes bytes) +ErrorOr DeflateCompressor::read(Bytes) +{ + return Error::from_errno(EBADF); +} + +ErrorOr DeflateCompressor::write(ReadonlyBytes bytes) { VERIFY(!m_finished); @@ -462,21 +468,25 @@ size_t DeflateCompressor::write(ReadonlyBytes bytes) m_pending_block_size += n_written; if (m_pending_block_size == block_size) - flush(); + TRY(flush()); - return n_written + write(bytes.slice(n_written)); + return n_written + TRY(write(bytes.slice(n_written))); } -bool DeflateCompressor::write_or_error(ReadonlyBytes bytes) +bool DeflateCompressor::is_eof() const { - if (write(bytes) < bytes.size()) { - set_fatal_error(); - return false; - } - return true; } +bool DeflateCompressor::is_open() const +{ + return m_output_stream->is_open(); +} + +void DeflateCompressor::close() +{ +} + // Knuth's multiplicative hash on 4 bytes u16 DeflateCompressor::hash_sequence(u8 const* bytes) { @@ -730,7 +740,7 @@ size_t DeflateCompressor::huffman_block_length(Array c size_t DeflateCompressor::uncompressed_block_length() { - auto padding = 8 - ((m_output_stream.bit_offset() + 3) % 8); + auto padding = 8 - ((m_output_stream->bit_offset() + 3) % 8); // 3 bit block header + align to byte + 2 * 16 bit length fields + block contents return 3 + padding + (2 * 16) + m_pending_block_size * 8; } @@ -765,25 +775,26 @@ size_t DeflateCompressor::dynamic_block_length(Array c return length + huffman_block_length(literal_bit_lengths, distance_bit_lengths); } -void DeflateCompressor::write_huffman(CanonicalCode const& literal_code, Optional const& distance_code) +ErrorOr DeflateCompressor::write_huffman(CanonicalCode const& literal_code, Optional const& distance_code) { auto has_distances = distance_code.has_value(); for (size_t i = 0; i < m_pending_symbol_size; i++) { if (m_symbol_buffer[i].distance == 0) { - literal_code.write_symbol(m_output_stream, m_symbol_buffer[i].literal); + TRY(literal_code.write_symbol(*m_output_stream, m_symbol_buffer[i].literal)); continue; } VERIFY(has_distances); auto symbol = length_to_symbol[m_symbol_buffer[i].length]; - literal_code.write_symbol(m_output_stream, symbol); + TRY(literal_code.write_symbol(*m_output_stream, symbol)); // Emit extra bits if needed - m_output_stream.write_bits(m_symbol_buffer[i].length - packed_length_symbols[symbol - 257].base_length, packed_length_symbols[symbol - 257].extra_bits); + TRY(m_output_stream->write_bits(m_symbol_buffer[i].length - packed_length_symbols[symbol - 257].base_length, packed_length_symbols[symbol - 257].extra_bits)); auto base_distance = distance_to_base(m_symbol_buffer[i].distance); - distance_code.value().write_symbol(m_output_stream, base_distance); + TRY(distance_code.value().write_symbol(*m_output_stream, base_distance)); // Emit extra bits if needed - m_output_stream.write_bits(m_symbol_buffer[i].distance - packed_distances[base_distance].base_distance, packed_distances[base_distance].extra_bits); + TRY(m_output_stream->write_bits(m_symbol_buffer[i].distance - packed_distances[base_distance].base_distance, packed_distances[base_distance].extra_bits)); } + return {}; } size_t DeflateCompressor::encode_huffman_lengths(Array const& lengths, size_t lengths_count, Array& encoded_lengths) @@ -854,65 +865,62 @@ size_t DeflateCompressor::encode_block_lengths(Array c return encode_huffman_lengths(all_lengths, lengths_count, encoded_lengths); } -void DeflateCompressor::write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional const& distance_code, size_t distance_code_count, Array const& code_lengths_bit_lengths, size_t code_length_count, Array const& encoded_lengths, size_t encoded_lengths_count) +ErrorOr DeflateCompressor::write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional const& distance_code, size_t distance_code_count, Array const& code_lengths_bit_lengths, size_t code_length_count, Array const& encoded_lengths, size_t encoded_lengths_count) { - m_output_stream.write_bits(literal_code_count - 257, 5); - m_output_stream.write_bits(distance_code_count - 1, 5); - m_output_stream.write_bits(code_length_count - 4, 4); + TRY(m_output_stream->write_bits(literal_code_count - 257, 5)); + TRY(m_output_stream->write_bits(distance_code_count - 1, 5)); + TRY(m_output_stream->write_bits(code_length_count - 4, 4)); for (size_t i = 0; i < code_length_count; i++) { - m_output_stream.write_bits(code_lengths_bit_lengths[code_lengths_code_lengths_order[i]], 3); + TRY(m_output_stream->write_bits(code_lengths_bit_lengths[code_lengths_code_lengths_order[i]], 3)); } auto code_lengths_code = CanonicalCode::from_bytes(code_lengths_bit_lengths); VERIFY(code_lengths_code.has_value()); for (size_t i = 0; i < encoded_lengths_count; i++) { auto encoded_length = encoded_lengths[i]; - code_lengths_code->write_symbol(m_output_stream, encoded_length.symbol); + TRY(code_lengths_code->write_symbol(*m_output_stream, encoded_length.symbol)); if (encoded_length.symbol == deflate_special_code_length_copy) { - m_output_stream.write_bits(encoded_length.count - 3, 2); + TRY(m_output_stream->write_bits(encoded_length.count - 3, 2)); } else if (encoded_length.symbol == deflate_special_code_length_zeros) { - m_output_stream.write_bits(encoded_length.count - 3, 3); + TRY(m_output_stream->write_bits(encoded_length.count - 3, 3)); } else if (encoded_length.symbol == deflate_special_code_length_long_zeros) { - m_output_stream.write_bits(encoded_length.count - 11, 7); + TRY(m_output_stream->write_bits(encoded_length.count - 11, 7)); } } - write_huffman(literal_code, distance_code); + TRY(write_huffman(literal_code, distance_code)); + return {}; } -void DeflateCompressor::flush() +ErrorOr DeflateCompressor::flush() { - if (m_output_stream.handle_any_error()) { - set_fatal_error(); - return; - } - - m_output_stream.write_bit(m_finished); + TRY(m_output_stream->write_bits(m_finished, 1)); // if this is just an empty block to signify the end of the deflate stream use the smallest block possible (10 bits total) if (m_pending_block_size == 0) { - VERIFY(m_finished); // we shouldn't be writing empty blocks unless this is the final one - m_output_stream.write_bits(0b01, 2); // fixed huffman codes - m_output_stream.write_bits(0b0000000, 7); // end of block symbol - m_output_stream.align_to_byte_boundary(); - return; + VERIFY(m_finished); // we shouldn't be writing empty blocks unless this is the final one + TRY(m_output_stream->write_bits(0b01u, 2)); // fixed huffman codes + TRY(m_output_stream->write_bits(0b0000000u, 7)); // end of block symbol + TRY(m_output_stream->align_to_byte_boundary()); + return {}; } - auto write_uncompressed = [&]() { - m_output_stream.write_bits(0b00, 2); // no compression - m_output_stream.align_to_byte_boundary(); + auto write_uncompressed = [&]() -> ErrorOr { + TRY(m_output_stream->write_bits(0b00u, 2)); // no compression + TRY(m_output_stream->align_to_byte_boundary()); LittleEndian len = m_pending_block_size; - m_output_stream << len; + TRY(m_output_stream->write_entire_buffer(len.bytes())); LittleEndian nlen = ~m_pending_block_size; - m_output_stream << nlen; - m_output_stream.write_or_error(pending_block().slice(0, m_pending_block_size)); + TRY(m_output_stream->write_entire_buffer(nlen.bytes())); + TRY(m_output_stream->write_entire_buffer(pending_block().slice(0, m_pending_block_size))); + return {}; }; if (m_compression_level == CompressionLevel::STORE) { // disabled compression fast path - write_uncompressed(); + TRY(write_uncompressed()); m_pending_block_size = 0; - return; + return {}; } // The following implementation of lz77 compression and huffman encoding is based on the reference implementation by Hans Wennborg https://www.hanshq.net/zip.html @@ -956,19 +964,21 @@ void DeflateCompressor::flush() // If the compression somehow didn't reduce the size enough, just write out the block uncompressed as it allows for much faster decompression if (uncompressed_size <= min(fixed_huffman_size, dynamic_huffman_size)) { - write_uncompressed(); - } else if (fixed_huffman_size <= dynamic_huffman_size) { // If the fixed and dynamic huffman codes come out the same size, prefer the fixed version, as it takes less time to decode - m_output_stream.write_bits(0b01, 2); // fixed huffman codes - write_huffman(CanonicalCode::fixed_literal_codes(), CanonicalCode::fixed_distance_codes()); + TRY(write_uncompressed()); + } else if (fixed_huffman_size <= dynamic_huffman_size) { + // If the fixed and dynamic huffman codes come out the same size, prefer the fixed version, as it takes less time to decode fixed huffman codes. + TRY(m_output_stream->write_bits(0b01u, 2)); + TRY(write_huffman(CanonicalCode::fixed_literal_codes(), CanonicalCode::fixed_distance_codes())); } else { - m_output_stream.write_bits(0b10, 2); // dynamic huffman codes + // dynamic huffman codes + TRY(m_output_stream->write_bits(0b10u, 2)); auto literal_code = CanonicalCode::from_bytes(dynamic_literal_bit_lengths); VERIFY(literal_code.has_value()); auto distance_code = CanonicalCode::from_bytes(dynamic_distance_bit_lengths); - write_dynamic_huffman(literal_code.value(), literal_code_count, distance_code, distance_code_count, code_lengths_bit_lengths, code_lengths_count, encoded_lengths, encoded_lengths_count); + TRY(write_dynamic_huffman(literal_code.value(), literal_code_count, distance_code, distance_code_count, code_lengths_bit_lengths, code_lengths_count, encoded_lengths, encoded_lengths_count)); } if (m_finished) - m_output_stream.align_to_byte_boundary(); + TRY(m_output_stream->align_to_byte_boundary()); // reset all block specific members m_pending_block_size = 0; @@ -977,28 +987,30 @@ void DeflateCompressor::flush() m_distance_frequencies.fill(0); // On the final block this copy will potentially produce an invalid search window, but since its the final block we dont care pending_block().copy_trimmed_to({ m_rolling_window, block_size }); + + return {}; } -void DeflateCompressor::final_flush() +ErrorOr DeflateCompressor::final_flush() { VERIFY(!m_finished); m_finished = true; - flush(); + TRY(flush()); + return {}; } -Optional DeflateCompressor::compress_all(ReadonlyBytes bytes, CompressionLevel compression_level) +ErrorOr DeflateCompressor::compress_all(ReadonlyBytes bytes, CompressionLevel compression_level) { - DuplexMemoryStream output_stream; - DeflateCompressor deflate_stream { output_stream, compression_level }; + auto output_stream = TRY(try_make()); + DeflateCompressor deflate_stream { Core::Stream::Handle(*output_stream), compression_level }; - deflate_stream.write_or_error(bytes); + TRY(deflate_stream.write_entire_buffer(bytes)); + TRY(deflate_stream.final_flush()); - deflate_stream.final_flush(); + auto buffer = TRY(ByteBuffer::create_uninitialized(output_stream->used_buffer_size())); + TRY(output_stream->read_entire_buffer(buffer)); - if (deflate_stream.handle_any_error()) - return {}; - - return output_stream.copy_into_contiguous_buffer(); + return buffer; } } diff --git a/Userland/Libraries/LibCompress/Deflate.h b/Userland/Libraries/LibCompress/Deflate.h index f94648cb589..8cdfaee287f 100644 --- a/Userland/Libraries/LibCompress/Deflate.h +++ b/Userland/Libraries/LibCompress/Deflate.h @@ -22,7 +22,7 @@ class CanonicalCode { public: CanonicalCode() = default; ErrorOr read_symbol(Core::Stream::LittleEndianInputBitStream&) const; - void write_symbol(OutputBitStream&, u32) const; + ErrorOr write_symbol(Core::Stream::LittleEndianOutputBitStream&, u32) const; static CanonicalCode const& fixed_literal_codes(); static CanonicalCode const& fixed_distance_codes(); @@ -104,7 +104,7 @@ private: CircularDuplexStream<32 * KiB> m_output_stream; }; -class DeflateCompressor final : public OutputStream { +class DeflateCompressor final : public Core::Stream::Stream { public: static constexpr size_t block_size = 32 * KiB - 1; // TODO: this can theoretically be increased to 64 KiB - 2 static constexpr size_t window_size = block_size * 2; @@ -139,14 +139,17 @@ public: BEST // WARNING: this one can take an unreasonable amount of time! }; - DeflateCompressor(OutputStream&, CompressionLevel = CompressionLevel::GOOD); + DeflateCompressor(Core::Stream::Handle, CompressionLevel = CompressionLevel::GOOD); ~DeflateCompressor(); - size_t write(ReadonlyBytes) override; - bool write_or_error(ReadonlyBytes) override; - void final_flush(); + virtual ErrorOr read(Bytes) override; + virtual ErrorOr write(ReadonlyBytes) override; + virtual bool is_eof() const override; + virtual bool is_open() const override; + virtual void close() override; + ErrorOr final_flush(); - static Optional compress_all(ReadonlyBytes bytes, CompressionLevel = CompressionLevel::GOOD); + static ErrorOr compress_all(ReadonlyBytes bytes, CompressionLevel = CompressionLevel::GOOD); private: Bytes pending_block() { return { m_rolling_window + block_size, block_size }; } @@ -166,20 +169,20 @@ private: template static void generate_huffman_lengths(Array& lengths, Array const& frequencies, size_t max_bit_length, u16 frequency_cap = UINT16_MAX); size_t huffman_block_length(Array const& literal_bit_lengths, Array const& distance_bit_lengths); - void write_huffman(CanonicalCode const& literal_code, Optional const& distance_code); + ErrorOr write_huffman(CanonicalCode const& literal_code, Optional const& distance_code); static size_t encode_huffman_lengths(Array const& lengths, size_t lengths_count, Array& encoded_lengths); size_t encode_block_lengths(Array const& literal_bit_lengths, Array const& distance_bit_lengths, Array& encoded_lengths, size_t& literal_code_count, size_t& distance_code_count); - void write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional const& distance_code, size_t distance_code_count, Array const& code_lengths_bit_lengths, size_t code_length_count, Array const& encoded_lengths, size_t encoded_lengths_count); + ErrorOr write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional const& distance_code, size_t distance_code_count, Array const& code_lengths_bit_lengths, size_t code_length_count, Array const& encoded_lengths, size_t encoded_lengths_count); size_t uncompressed_block_length(); size_t fixed_block_length(); size_t dynamic_block_length(Array const& literal_bit_lengths, Array const& distance_bit_lengths, Array const& code_lengths_bit_lengths, Array const& code_lengths_frequencies, size_t code_lengths_count); - void flush(); + ErrorOr flush(); bool m_finished { false }; CompressionLevel m_compression_level; CompressionConstants m_compression_constants; - OutputBitStream m_output_stream; + NonnullOwnPtr m_output_stream; u8 m_rolling_window[window_size]; size_t m_pending_block_size { 0 }; diff --git a/Userland/Libraries/LibCompress/Gzip.cpp b/Userland/Libraries/LibCompress/Gzip.cpp index 4f2426d04c0..5675b30eb4f 100644 --- a/Userland/Libraries/LibCompress/Gzip.cpp +++ b/Userland/Libraries/LibCompress/Gzip.cpp @@ -194,11 +194,9 @@ ErrorOr GzipCompressor::write(ReadonlyBytes bytes) header.extra_flags = 3; // DEFLATE sets 2 for maximum compression and 4 for minimum compression header.operating_system = 3; // unix TRY(m_output_stream->write_entire_buffer({ &header, sizeof(header) })); - Core::Stream::WrapInAKOutputStream wrapped_stream { *m_output_stream }; - DeflateCompressor compressed_stream { wrapped_stream }; - if (!compressed_stream.write_or_error(bytes)) - return Error::from_string_literal("Underlying DeflateCompressor indicated an error"); - compressed_stream.final_flush(); + DeflateCompressor compressed_stream { Core::Stream::Handle(*m_output_stream) }; + TRY(compressed_stream.write_entire_buffer(bytes)); + TRY(compressed_stream.final_flush()); Crypto::Checksum::CRC32 crc32; crc32.update(bytes); LittleEndian digest = crc32.digest(); diff --git a/Userland/Libraries/LibCompress/Zlib.cpp b/Userland/Libraries/LibCompress/Zlib.cpp index 40c67acf8c1..83924b8d5d8 100644 --- a/Userland/Libraries/LibCompress/Zlib.cpp +++ b/Userland/Libraries/LibCompress/Zlib.cpp @@ -81,10 +81,9 @@ ErrorOr> ZlibCompressor::construct(Core::Stream::H } ZlibCompressor::ZlibCompressor(Core::Stream::Handle stream, ZlibCompressionLevel compression_level) - : m_ak_output_stream(make(*stream)) - , m_output_stream(move(stream)) + : m_output_stream(move(stream)) // FIXME: Find a way to compress with Deflate's "Best" compression level. - , m_compressor(make(*m_ak_output_stream, static_cast(compression_level))) + , m_compressor(make(Core::Stream::Handle(*m_output_stream), static_cast(compression_level))) { } @@ -126,7 +125,7 @@ ErrorOr ZlibCompressor::write(ReadonlyBytes bytes) { VERIFY(!m_finished); - size_t n_written = m_compressor->write(bytes); + size_t n_written = TRY(m_compressor->write(bytes)); m_adler32_checksum.update(bytes.trim(n_written)); return n_written; } @@ -150,7 +149,7 @@ ErrorOr ZlibCompressor::finish() VERIFY(!m_finished); if (is(m_compressor.ptr())) - static_cast(m_compressor.ptr())->final_flush(); + TRY(static_cast(m_compressor.ptr())->final_flush()); NetworkOrdered adler_sum = m_adler32_checksum.digest(); TRY(m_output_stream->write(adler_sum.bytes())); diff --git a/Userland/Libraries/LibCompress/Zlib.h b/Userland/Libraries/LibCompress/Zlib.h index 56b663773ed..9d88cd966d2 100644 --- a/Userland/Libraries/LibCompress/Zlib.h +++ b/Userland/Libraries/LibCompress/Zlib.h @@ -80,10 +80,8 @@ private: ErrorOr write_header(ZlibCompressionMethod, ZlibCompressionLevel); bool m_finished { false }; - // FIXME: Remove this once DeflateCompressor is ported to Core::Stream. - NonnullOwnPtr m_ak_output_stream; Core::Stream::Handle m_output_stream; - NonnullOwnPtr m_compressor; + NonnullOwnPtr m_compressor; Crypto::Checksum::Adler32 m_adler32_checksum; }; diff --git a/Userland/Utilities/zip.cpp b/Userland/Utilities/zip.cpp index 2fcb15b9c54..3e44c26da40 100644 --- a/Userland/Utilities/zip.cpp +++ b/Userland/Utilities/zip.cpp @@ -59,7 +59,7 @@ ErrorOr serenity_main(Main::Arguments arguments) member.name = TRY(String::from_deprecated_string(canonicalized_path)); auto deflate_buffer = Compress::DeflateCompressor::compress_all(file_buffer); - if (deflate_buffer.has_value() && deflate_buffer.value().size() < file_buffer.size()) { + if (!deflate_buffer.is_error() && deflate_buffer.value().size() < file_buffer.size()) { member.compressed_data = deflate_buffer.value().bytes(); member.compression_method = Archive::ZipCompressionMethod::Deflate; auto compression_ratio = (double)deflate_buffer.value().size() / file_buffer.size();