LibWeb: Implement RSAOAEP.decrypt()

Author: https://github.com/stelar7
Commit: https://github.com/LadybirdBrowser/ladybird/commit/23fc04d264b
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/1976

Tests/LibCrypto/TestOAEP.cpp

@@ -61,11 +61,6 @@ TEST_CASE(test_oaep)
 
     EXPECT_EQ(expected, result);
 
-    auto maybe_decode = Crypto::Padding::OAEP::decode<Crypto::Hash::SHA1, Crypto::Hash::MGF>(result, params);
-    auto decoded = maybe_decode.release_value();
-
-    EXPECT_EQ(message, decoded);
-
     u8 n_bytes[128] {
         0xbb, 0xf8, 0x2f, 0x09, 0x06, 0x82, 0xce, 0x9c, 0x23, 0x38, 0xac, 0x2b, 0x9d, 0xa8, 0x71, 0xf7,
         0x36, 0x8d, 0x07, 0xee, 0xd4, 0x10, 0x43, 0xa4, 0x40, 0xd6, 0xb6, 0xf0, 0x74, 0x54, 0xf5, 0x1f,

Userland/Libraries/LibCrypto/PK/RSA.h

@@ -94,8 +94,8 @@ public:
 
     static RSAPrivateKey from_crt(Integer n, Integer e, Integer p, Integer q, Integer dp, Integer dq, Integer qinv)
     {
-        auto lambda = NumberTheory::LCM(p.minus(1), q.minus(1));
-        auto d = NumberTheory::ModularInverse(e, lambda);
+        auto phi = p.minus(1).multiplied_by(q.minus(1));
+        auto d = NumberTheory::ModularInverse(e, phi);
 
         return { n, d, e, p, q, dp, dq, qinv };
     }
@@ -231,6 +231,7 @@ public:
     PublicKeyType const& public_key() const { return m_public_key; }
 
     void set_public_key(PublicKeyType const& key) { m_public_key = key; }
+    void set_private_key(PrivateKeyType const& key) { m_private_key = key; }
 };
 
 class RSA_PKCS1_EME : public RSA {

Userland/Libraries/LibCrypto/Padding/OAEP.h

@@ -87,6 +87,7 @@ public:
         auto k = rsa_modulus_n;
         auto h_len = HashFunction::digest_size();
         auto max_message_size = k - (2 * h_len) - 2;
+
         if (m_len > max_message_size)
             return Error::from_string_view("message too long"sv);
 
@@ -196,6 +197,78 @@ public:
         // 11. Output M.
         return message;
     }
+
+    // https://www.rfc-editor.org/rfc/rfc3447#section-7.1.2
+    template<typename HashFunction, typename MaskGenerationFunction>
+    static ErrorOr<ByteBuffer> eme_decode(ReadonlyBytes encoded_message, ReadonlyBytes label, u32 rsa_modulus_n)
+    {
+        auto h_len = HashFunction::digest_size();
+        auto k = rsa_modulus_n;
+
+        // 1. If the label L is not provided, let L be the empty string.
+        // Let lHash = Hash(L), an octet string of length hLen (see the note in Section 7.1.1).
+        HashFunction hash;
+        hash.update(label);
+        auto digest = hash.digest();
+        auto l_hash = digest.bytes();
+
+        // 2. Separate the encoded message EM into
+        //    a single octet Y,
+        //    an octet string maskedSeed of length hLen,
+        //    and an octet string maskedDB of length k - hLen - 1
+        //    as EM = Y || maskedSeed || maskedDB.
+        auto y = encoded_message[0];
+        auto masked_seed = encoded_message.slice(1, h_len);
+        auto masked_db = encoded_message.slice(h_len + 1, k - h_len - 1);
+
+        // 3. Let seedMask = MGF(maskedDB, hLen).
+        auto seed_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(masked_db, h_len));
+
+        // 4. Let seed = maskedSeed \xor seedMask.
+        auto seed = TRY(ByteBuffer::xor_buffers(masked_seed, seed_mask));
+
+        // 5. Let dbMask = MGF(seed, k - hLen - 1).
+        auto db_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(seed, k - h_len - 1));
+
+        // 6. Let DB = maskedDB \xor dbMask.
+        auto db = TRY(ByteBuffer::xor_buffers(masked_db, db_mask));
+
+        // 7. Separate DB into
+        // an octet string lHash' of length hLen,
+        // a (possibly empty) padding string PS consisting of octets withhexadecimal value 0x00,
+        // and a message M
+        // as DB = lHash' || PS || 0x01 || M.
+        auto l_hash_prime = TRY(db.slice(0, h_len));
+
+        size_t i = h_len;
+        for (; i < db.size(); ++i) {
+            if (db[i] == 0x01)
+                break;
+        }
+
+        auto message = TRY(db.slice(i + 1, db.size() - i - 1));
+
+        // NOTE: We have to make sure to do all these steps before returning an error due to timing attacks
+        bool is_valid = true;
+
+        // If there is no octet with hexadecimal value 0x01 to separate PS from M,
+        if (i == db.size())
+            is_valid = false;
+
+        // if lHash does not equal lHash',
+        if (l_hash_prime.span() != l_hash)
+            is_valid = false;
+
+        // if Y is nonzero, output "decryption error" and stop.
+        if (y != 0x00)
+            is_valid = false;
+
+        if (!is_valid)
+            return Error::from_string_view("decryption error"sv);
+
+        // 8. Output the message M.
+        return message;
+    }
 };
 
 }

Userland/Libraries/LibWeb/Crypto/CryptoAlgorithms.cpp

@@ -550,17 +550,43 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<JS::ArrayBuffer>> RSAOAEP::decrypt(Algorith
         return WebIDL::InvalidAccessError::create(realm, "Key is not a private key"_string);
 
     // 2. Let label be the contents of the label member of normalizedAlgorithm or the empty octet string if the label member of normalizedAlgorithm is not present.
-    [[maybe_unused]] auto const& label = normalized_algorithm.label;
+    auto const& label = normalized_algorithm.label;
+
+    auto const& handle = key->handle();
+    auto private_key = handle.get<::Crypto::PK::RSAPrivateKey<>>();
+    auto hash = TRY(verify_cast<RsaHashedKeyAlgorithm>(*key->algorithm()).hash().name(vm));
 
     // 3. Perform the decryption operation defined in Section 7.1 of [RFC3447] with the key represented by key as the recipient's RSA private key,
     //    the contents of ciphertext as the ciphertext to be decrypted, C, and label as the label, L, and with the hash function specified by the hash attribute
     //    of the [[algorithm]] internal slot of key as the Hash option and MGF1 (defined in Section B.2.1 of [RFC3447]) as the MGF option.
+    auto rsa = ::Crypto::PK::RSA {};
+    rsa.set_private_key(private_key);
+    u32 private_key_length = private_key.length();
+
+    auto padding = TRY_OR_THROW_OOM(vm, ByteBuffer::create_uninitialized(private_key_length));
+    auto padding_bytes = padding.bytes();
+    rsa.decrypt(ciphertext, padding_bytes);
+
+    auto error_message = MUST(String::formatted("Invalid hash function '{}'", hash));
+    ErrorOr<ByteBuffer> maybe_plaintext = Error::from_string_view(error_message.bytes_as_string_view());
+    if (hash.equals_ignoring_ascii_case("SHA-1"sv)) {
+        maybe_plaintext = ::Crypto::Padding::OAEP::eme_decode<::Crypto::Hash::SHA1, ::Crypto::Hash::MGF>(padding, label, private_key_length);
+    } else if (hash.equals_ignoring_ascii_case("SHA-256"sv)) {
+        maybe_plaintext = ::Crypto::Padding::OAEP::eme_decode<::Crypto::Hash::SHA256, ::Crypto::Hash::MGF>(padding, label, private_key_length);
+    } else if (hash.equals_ignoring_ascii_case("SHA-384"sv)) {
+        maybe_plaintext = ::Crypto::Padding::OAEP::eme_decode<::Crypto::Hash::SHA384, ::Crypto::Hash::MGF>(padding, label, private_key_length);
+    } else if (hash.equals_ignoring_ascii_case("SHA-512"sv)) {
+        maybe_plaintext = ::Crypto::Padding::OAEP::eme_decode<::Crypto::Hash::SHA512, ::Crypto::Hash::MGF>(padding, label, private_key_length);
+    }
 
     // 4. If performing the operation results in an error, then throw an OperationError.
+    if (maybe_plaintext.is_error()) {
+        auto error_message = MUST(String::from_utf8(maybe_plaintext.error().string_literal()));
+        return WebIDL::OperationError::create(realm, error_message);
+    }
 
     // 5. Let plaintext the value M that results from performing the operation.
-    // FIXME: Actually decrypt the data
-    auto plaintext = TRY_OR_THROW_OOM(vm, ByteBuffer::copy(ciphertext));
+    auto plaintext = maybe_plaintext.release_value();
 
     // 6. Return the result of creating an ArrayBuffer containing plaintext.
     return JS::ArrayBuffer::create(realm, move(plaintext));