LibCrypto: Implement OAEP decode from newer spec

Author: https://github.com/stelar7
Commit: https://github.com/LadybirdBrowser/ladybird/commit/378808f6ba9
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/1976

Tests/LibCrypto/TestOAEP.cpp

@@ -61,6 +61,11 @@ TEST_CASE(test_oaep)
 
     EXPECT_EQ(expected, result);
 
+    auto maybe_decode = Crypto::Padding::OAEP::decode<Crypto::Hash::SHA1, Crypto::Hash::MGF>(result, params);
+    auto decoded = maybe_decode.release_value();
+
+    EXPECT_EQ(message, decoded);
+
     u8 n_bytes[128] {
         0xbb, 0xf8, 0x2f, 0x09, 0x06, 0x82, 0xce, 0x9c, 0x23, 0x38, 0xac, 0x2b, 0x9d, 0xa8, 0x71, 0xf7,
         0x36, 0x8d, 0x07, 0xee, 0xd4, 0x10, 0x43, 0xa4, 0x40, 0xd6, 0xb6, 0xf0, 0x74, 0x54, 0xf5, 0x1f,

Userland/Libraries/LibCrypto/Padding/OAEP.h

@@ -134,6 +134,68 @@ public:
         // 12. Output EM.
         return em;
     }
+
+    // https://datatracker.ietf.org/doc/html/rfc2437#section-9.1.1.2
+    template<typename HashFunction, typename MaskGenerationFunction>
+    static ErrorOr<ByteBuffer> decode(ReadonlyBytes encoded_message, ReadonlyBytes parameters)
+    {
+        // FIXME: 1. If the length of P is greater than the input limitation for the
+        //           hash function (2^61-1 octets for SHA-1) then output "parameter string
+        //           too long" and stop.
+
+        // 2. If ||EM|| < 2hLen+1, then output "decoding error" and stop.
+        auto h_len = HashFunction::digest_size();
+        auto max_message_size = (2 * h_len) + 1;
+        if (encoded_message.size() < max_message_size)
+            return Error::from_string_view("decoding error"sv);
+
+        // 3. Let maskedSeed be the first hLen octets of EM and let maskedDB be the remaining ||EM|| - hLen octets.
+        auto masked_seed = encoded_message.slice(0, h_len);
+        auto masked_db = encoded_message.slice(h_len, encoded_message.size() - h_len);
+
+        // 4. Let seedMask = MGF(maskedDB, hLen).
+        auto seed_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(masked_db, h_len));
+
+        // 5. Let seed = maskedSeed \xor seedMask.
+        auto seed = TRY(ByteBuffer::xor_buffers(masked_seed, seed_mask));
+
+        // 6. Let dbMask = MGF(seed, ||EM|| - hLen).
+        auto db_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(seed, encoded_message.size() - h_len));
+
+        // 7. Let DB = maskedDB \xor dbMask.
+        auto db = TRY(ByteBuffer::xor_buffers(masked_db, db_mask));
+
+        // 8. Let pHash = Hash(P), an octet string of length hLen.
+        HashFunction hash;
+        hash.update(parameters);
+        auto digest = hash.digest();
+        auto p_hash = digest.bytes();
+
+        // 9. Separate DB into an octet string pHash' consisting of the first hLen octets of DB,
+        //    a (possibly empty) octet string PS consisting of consecutive zero octets following pHash',
+        //    and a message M as: DB = pHash' || PS || 01 || M
+        auto p_hash_prime = TRY(db.slice(0, h_len));
+
+        size_t i = h_len;
+        for (; i < db.size(); ++i) {
+            if (db[i] == 0x01)
+                break;
+        }
+
+        //    If there is no 01 octet to separate PS from M, output "decoding error" and stop.
+        if (i == db.size())
+            return Error::from_string_view("decoding error"sv);
+
+        auto ps = TRY(db.slice(h_len, i - h_len));
+        auto message = TRY(db.slice(i + 1, db.size() - i - 1));
+
+        // 10. If pHash' does not equal pHash, output "decoding error" and stop.
+        if (p_hash_prime.span() != p_hash)
+            return Error::from_string_view("decoding error"sv);
+
+        // 11. Output M.
+        return message;
+    }
 };
 
 }