LibELF: Add support for IFUNCs

IFUNC is a GNU extension to the ELF standard that allows a function to
have multiple implementations. A resolver function has to be called at
load time to choose the right one to use. The PLT will contain the entry
to the resolved function, so branching and more indirect jumps can be
avoided at run-time.

This mechanism is usually used when a routine can be made faster using
CPU features that are available in only some models, and a fallback
implementation has to exist for others.

We will use this feature to have two separate memset implementations for
CPUs with and without ERMS (Enhanced REP MOVSB/STOSB) support.

Userland/Libraries/LibC/elf.h

@@ -381,6 +381,7 @@ typedef struct {
 #define STT_SECTION 3 /* section */
 #define STT_FILE 4    /* file */
 #define STT_TLS 6     /* thread local storage */
+#define STT_GNU_IFUNC 10
 #define STT_LOPROC 13 /* reserved range for processor */
 #define STT_HIPROC 15 /*  specific symbol types */
 
@@ -812,6 +813,7 @@ struct elf_args {
 #define R_386_RELATIVE 8   /* Base address + Addned */
 #define R_386_TLS_TPOFF 14 /* Negative offset into the static TLS storage */
 #define R_386_TLS_TPOFF32 37
+#define R_386_IRELATIVE 42 /* PLT entry resolved indirectly at runtime */
 
 #define R_X86_64_NONE 0
 #define R_X86_64_64 1
@@ -819,3 +821,4 @@ struct elf_args {
 #define R_X86_64_JUMP_SLOT 7
 #define R_X86_64_RELATIVE 8
 #define R_X86_64_TPOFF64 18
+#define R_X86_64_IRELATIVE 37

Userland/Libraries/LibELF/DynamicLinker.cpp

@@ -469,10 +469,12 @@ static Result<void*, DlErrorMessage> __dlsym(void* handle, char const* symbol_na
         symbol = DynamicLinker::lookup_global_symbol(symbol_name);
     }
 
-    if (symbol.has_value())
-        return symbol.value().address.as_ptr();
+    if (!symbol.has_value())
+        return DlErrorMessage { String::formatted("Symbol {} not found", symbol_name) };
 
-    return DlErrorMessage { String::formatted("Symbol {} not found", symbol_name) };
+    if (symbol.value().type == STT_GNU_IFUNC)
+        return (void*)reinterpret_cast<DynamicObject::IfuncResolver>(symbol.value().address.as_ptr())();
+    return symbol.value().address.as_ptr();
 }
 
 static Result<void, DlErrorMessage> __dladdr(void* addr, Dl_info* info)

Userland/Libraries/LibELF/DynamicLoader.cpp

@@ -2,6 +2,7 @@
  * Copyright (c) 2019-2020, Andrew Kaster <akaster@serenityos.org>
  * Copyright (c) 2020, Itamar S. <itamar8910@gmail.com>
  * Copyright (c) 2021, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2022, Daniel Bertalan <dani@danielbertalan.dev>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -177,6 +178,15 @@ bool DynamicLoader::load_stage_2(unsigned flags)
                 return false;
             }
         }
+    } else {
+        // .text needs to be executable while we process relocations because it might contain IFUNC resolvers.
+        // We don't allow IFUNC resolvers in objects with textrels.
+        for (auto& text_segment : m_text_segments) {
+            if (mprotect(text_segment.address().as_ptr(), text_segment.size(), PROT_READ | PROT_EXEC) < 0) {
+                perror("mprotect .text: PROT_READ | PROT_EXEC");
+                return false;
+            }
+        }
     }
     do_main_relocations();
     return true;
@@ -210,9 +220,12 @@ Result<NonnullRefPtr<DynamicObject>, DlErrorMessage> DynamicLoader::load_stage_3
             setup_plt_trampoline();
     }
 
-    for (auto& text_segment : m_text_segments) {
-        if (mprotect(text_segment.address().as_ptr(), text_segment.size(), PROT_READ | PROT_EXEC) < 0) {
-            return DlErrorMessage { String::formatted("mprotect .text: PROT_READ | PROT_EXEC: {}", strerror(errno)) };
+    if (m_dynamic_object->has_text_relocations()) {
+        // If we don't have textrels, .text has already been made executable by this point in load_stage_2.
+        for (auto& text_segment : m_text_segments) {
+            if (mprotect(text_segment.address().as_ptr(), text_segment.size(), PROT_READ | PROT_EXEC) < 0) {
+                return DlErrorMessage { String::formatted("mprotect .text: PROT_READ | PROT_EXEC: {}", strerror(errno)) };
+            }
         }
     }
 
@@ -411,6 +424,10 @@ DynamicLoader::RelocationResult DynamicLoader::do_relocation(const ELF::DynamicO
     else
         patch_ptr = (FlatPtr*)(FlatPtr)relocation.offset();
 
+    auto call_ifunc_resolver = [](VirtualAddress address) {
+        return VirtualAddress { reinterpret_cast<DynamicObject::IfuncResolver>(address.get())() };
+    };
+
     switch (relocation.type()) {
 #if ARCH(I386)
     case R_386_NONE:
@@ -438,6 +455,8 @@ DynamicLoader::RelocationResult DynamicLoader::do_relocation(const ELF::DynamicO
             *patch_ptr = symbol_address.get() + relocation.addend();
         else
             *patch_ptr += symbol_address.get();
+        if (res.value().type == STT_GNU_IFUNC)
+            *patch_ptr = call_ifunc_resolver(VirtualAddress { *patch_ptr }).get();
         break;
     }
 #if ARCH(I386)
@@ -467,8 +486,11 @@ DynamicLoader::RelocationResult DynamicLoader::do_relocation(const ELF::DynamicO
             }
 
             symbol_location = VirtualAddress { (FlatPtr)0 };
-        } else
+        } else {
             symbol_location = res.value().address;
+            if (res.value().type == STT_GNU_IFUNC)
+                symbol_location = call_ifunc_resolver(symbol_location);
+        }
         VERIFY(symbol_location != m_dynamic_object->base_address());
         *patch_ptr = symbol_location.get();
         break;
@@ -500,6 +522,7 @@ DynamicLoader::RelocationResult DynamicLoader::do_relocation(const ELF::DynamicO
             auto res = lookup_symbol(symbol);
             if (!res.has_value())
                 break;
+            VERIFY(symbol.type() != STT_GNU_IFUNC);
             symbol_value = res.value().value;
             dynamic_object_of_symbol = res.value().dynamic_object;
         } else {
@@ -529,6 +552,19 @@ DynamicLoader::RelocationResult DynamicLoader::do_relocation(const ELF::DynamicO
         }
         break;
     }
+#if ARCH(I386)
+    case R_386_IRELATIVE: {
+#else
+    case R_X86_64_IRELATIVE: {
+#endif
+        VirtualAddress resolver;
+        if (relocation.addend_used())
+            resolver = m_dynamic_object->base_address().offset(relocation.addend());
+        else
+            resolver = m_dynamic_object->base_address().offset(*patch_ptr);
+        *patch_ptr = call_ifunc_resolver(resolver).get();
+        break;
+    }
     default:
         // Raise the alarm! Someone needs to implement this relocation type
         dbgln("Found a new exciting relocation type {}", relocation.type());
@@ -635,7 +671,7 @@ Optional<DynamicObject::SymbolLookupResult> DynamicLoader::lookup_symbol(const E
     if (symbol.is_undefined() || symbol.bind() == STB_WEAK)
         return DynamicLinker::lookup_global_symbol(symbol.name());
 
-    return DynamicObject::SymbolLookupResult { symbol.value(), symbol.size(), symbol.address(), symbol.bind(), &symbol.object() };
+    return DynamicObject::SymbolLookupResult { symbol.value(), symbol.size(), symbol.address(), symbol.bind(), symbol.type(), &symbol.object() };
 }
 
 } // end namespace ELF

Userland/Libraries/LibELF/DynamicObject.cpp

@@ -471,7 +471,7 @@ auto DynamicObject::lookup_symbol(HashSymbol const& symbol) const -> Optional<Sy
     auto symbol_result = result.value();
     if (symbol_result.is_undefined())
         return {};
-    return SymbolLookupResult { symbol_result.value(), symbol_result.size(), symbol_result.address(), symbol_result.bind(), this };
+    return SymbolLookupResult { symbol_result.value(), symbol_result.size(), symbol_result.address(), symbol_result.bind(), symbol_result.type(), this };
 }
 
 NonnullRefPtr<DynamicObject> DynamicObject::create(String const& filename, VirtualAddress base_address, VirtualAddress dynamic_section_address)
@@ -495,6 +495,9 @@ VirtualAddress DynamicObject::patch_plt_entry(u32 relocation_offset)
     auto result = DynamicLoader::lookup_symbol(symbol);
     if (result.has_value()) {
         symbol_location = result.value().address;
+
+        if (result.value().type == STT_GNU_IFUNC)
+            symbol_location = VirtualAddress { reinterpret_cast<IfuncResolver>(symbol_location.get())() };
     } else if (symbol.bind() != STB_WEAK) {
         dbgln("did not find symbol while doing relocations for library {}: {}", m_filename, symbol.name());
         VERIFY_NOT_REACHED();

Userland/Libraries/LibELF/DynamicObject.h

@@ -255,6 +255,7 @@ public:
     Symbol symbol(unsigned) const;
 
     typedef void (*InitializationFunction)();
+    typedef ElfW(Addr) (*IfuncResolver)();
 
     bool has_init_section() const { return m_init_offset != 0; }
     bool has_init_array_section() const { return m_init_array_offset != 0; }
@@ -322,6 +323,7 @@ public:
         size_t size { 0 };
         VirtualAddress address;
         unsigned bind { STB_LOCAL };
+        unsigned type { STT_FUNC };
         const ELF::DynamicObject* dynamic_object { nullptr }; // The object in which the symbol is defined
     };
 

Userland/Libraries/LibELF/Image.cpp

@@ -362,7 +362,7 @@ Optional<Image::Symbol> Image::find_demangled_function(StringView name) const
 {
     Optional<Image::Symbol> found;
     for_each_symbol([&](Image::Symbol const& symbol) {
-        if (symbol.type() != STT_FUNC)
+        if (symbol.type() != STT_FUNC && symbol.type() != STT_GNU_IFUNC)
             return IterationDecision::Continue;
         if (symbol.is_undefined())
             return IterationDecision::Continue;