DynamicLoader.cpp 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848
  1. /*
  2. * Copyright (c) 2019-2020, Andrew Kaster <akaster@serenityos.org>
  3. * Copyright (c) 2020, Itamar S. <itamar8910@gmail.com>
  4. * Copyright (c) 2021, Andreas Kling <kling@serenityos.org>
  5. * Copyright (c) 2022-2023, Daniel Bertalan <dani@danielbertalan.dev>
  6. *
  7. * SPDX-License-Identifier: BSD-2-Clause
  8. */
  9. #include <AK/Debug.h>
  10. #include <AK/Optional.h>
  11. #include <AK/QuickSort.h>
  12. #include <AK/StringBuilder.h>
  13. #include <LibELF/Arch/GenericDynamicRelocationType.h>
  14. #include <LibELF/DynamicLinker.h>
  15. #include <LibELF/DynamicLoader.h>
  16. #include <LibELF/Hashes.h>
  17. #include <LibELF/Validation.h>
  18. #include <assert.h>
  19. #include <bits/dlfcn_integration.h>
  20. #include <dlfcn.h>
  21. #include <errno.h>
  22. #include <stdio.h>
  23. #include <stdlib.h>
  24. #include <string.h>
  25. #include <sys/mman.h>
  26. #include <sys/stat.h>
  27. #include <unistd.h>
  28. #ifndef AK_OS_SERENITY
  29. static void* mmap_with_name(void* addr, size_t length, int prot, int flags, int fd, off_t offset, char const*)
  30. {
  31. return mmap(addr, length, prot, flags, fd, offset);
  32. }
  33. # define MAP_RANDOMIZED 0
  34. #endif
  35. #if ARCH(AARCH64)
  36. # define HAS_TLSDESC_SUPPORT
  37. extern "C" {
  38. void* __tlsdesc_static(void*);
  39. }
  40. #endif
  41. namespace ELF {
  42. Result<NonnullRefPtr<DynamicLoader>, DlErrorMessage> DynamicLoader::try_create(int fd, ByteString filepath)
  43. {
  44. VERIFY(filepath.starts_with('/'));
  45. struct stat stat;
  46. if (fstat(fd, &stat) < 0) {
  47. return DlErrorMessage { "DynamicLoader::try_create fstat" };
  48. }
  49. VERIFY(stat.st_size >= 0);
  50. auto size = static_cast<size_t>(stat.st_size);
  51. if (size < sizeof(Elf_Ehdr))
  52. return DlErrorMessage { ByteString::formatted("File {} has invalid ELF header", filepath) };
  53. ByteString file_mmap_name = ByteString::formatted("ELF_DYN: {}", filepath);
  54. auto* data = mmap_with_name(nullptr, size, PROT_READ, MAP_SHARED, fd, 0, file_mmap_name.characters());
  55. if (data == MAP_FAILED) {
  56. return DlErrorMessage { "DynamicLoader::try_create mmap" };
  57. }
  58. auto loader = adopt_ref(*new DynamicLoader(fd, move(filepath), data, size));
  59. if (!loader->is_valid())
  60. return DlErrorMessage { "ELF image validation failed" };
  61. return loader;
  62. }
  63. DynamicLoader::DynamicLoader(int fd, ByteString filepath, void* data, size_t size)
  64. : m_filepath(move(filepath))
  65. , m_file_size(size)
  66. , m_image_fd(fd)
  67. , m_file_data(data)
  68. {
  69. m_elf_image = adopt_own(*new ELF::Image((u8*)m_file_data, m_file_size));
  70. m_valid = validate();
  71. if (m_valid)
  72. find_tls_size_and_alignment();
  73. else
  74. dbgln("Image validation failed for file {}", m_filepath);
  75. }
  76. DynamicLoader::~DynamicLoader()
  77. {
  78. if (munmap(m_file_data, m_file_size) < 0) {
  79. perror("munmap");
  80. VERIFY_NOT_REACHED();
  81. }
  82. if (close(m_image_fd) < 0) {
  83. perror("close");
  84. VERIFY_NOT_REACHED();
  85. }
  86. }
  87. DynamicObject const& DynamicLoader::dynamic_object() const
  88. {
  89. if (!m_cached_dynamic_object) {
  90. VirtualAddress dynamic_section_address;
  91. image().for_each_program_header([&dynamic_section_address](auto program_header) {
  92. if (program_header.type() == PT_DYNAMIC) {
  93. dynamic_section_address = VirtualAddress(program_header.raw_data());
  94. }
  95. });
  96. VERIFY(!dynamic_section_address.is_null());
  97. m_cached_dynamic_object = ELF::DynamicObject::create(m_filepath, VirtualAddress(image().base_address()), dynamic_section_address);
  98. }
  99. return *m_cached_dynamic_object;
  100. }
  101. void DynamicLoader::find_tls_size_and_alignment()
  102. {
  103. image().for_each_program_header([this](auto program_header) {
  104. if (program_header.type() == PT_TLS) {
  105. m_tls_size_of_current_object = program_header.size_in_memory();
  106. auto alignment = program_header.alignment();
  107. VERIFY(!alignment || is_power_of_two(alignment));
  108. m_tls_alignment_of_current_object = alignment > 1 ? alignment : 0; // No need to reserve extra space for single byte alignment
  109. return IterationDecision::Break;
  110. }
  111. return IterationDecision::Continue;
  112. });
  113. }
  114. bool DynamicLoader::validate()
  115. {
  116. if (!image().is_valid())
  117. return false;
  118. auto* elf_header = (Elf_Ehdr*)m_file_data;
  119. if (!validate_elf_header(*elf_header, m_file_size))
  120. return false;
  121. auto result_or_error = validate_program_headers(*elf_header, m_file_size, { m_file_data, m_file_size });
  122. if (result_or_error.is_error() || !result_or_error.value())
  123. return false;
  124. return true;
  125. }
  126. RefPtr<DynamicObject> DynamicLoader::map()
  127. {
  128. if (m_dynamic_object) {
  129. // Already mapped.
  130. return nullptr;
  131. }
  132. if (!m_valid) {
  133. dbgln("DynamicLoader::map failed: image is invalid");
  134. return nullptr;
  135. }
  136. load_program_headers();
  137. VERIFY(!m_base_address.is_null());
  138. m_dynamic_object = DynamicObject::create(m_filepath, m_base_address, m_dynamic_section_address);
  139. m_dynamic_object->set_tls_offset(m_tls_offset);
  140. m_dynamic_object->set_tls_size(m_tls_size_of_current_object);
  141. return m_dynamic_object;
  142. }
  143. bool DynamicLoader::link(unsigned flags)
  144. {
  145. return load_stage_2(flags);
  146. }
  147. bool DynamicLoader::load_stage_2(unsigned flags)
  148. {
  149. VERIFY(flags & RTLD_GLOBAL);
  150. if (m_dynamic_object->has_text_relocations()) {
  151. dbgln("\033[33mWarning:\033[0m Dynamic object {} has text relocations", m_dynamic_object->filepath());
  152. for (auto& text_segment : m_text_segments) {
  153. VERIFY(text_segment.address().get() != 0);
  154. #ifndef AK_OS_MACOS
  155. // Remap this text region as private.
  156. if (mremap(text_segment.address().as_ptr(), text_segment.size(), text_segment.size(), MAP_PRIVATE) == MAP_FAILED) {
  157. perror("mremap .text: MAP_PRIVATE");
  158. return false;
  159. }
  160. #endif
  161. if (0 > mprotect(text_segment.address().as_ptr(), text_segment.size(), PROT_READ | PROT_WRITE)) {
  162. perror("mprotect .text: PROT_READ | PROT_WRITE"); // FIXME: dlerror?
  163. return false;
  164. }
  165. }
  166. } else {
  167. // .text needs to be executable while we process relocations because it might contain IFUNC resolvers.
  168. // We don't allow IFUNC resolvers in objects with textrels.
  169. for (auto& text_segment : m_text_segments) {
  170. if (mprotect(text_segment.address().as_ptr(), text_segment.size(), PROT_READ | PROT_EXEC) < 0) {
  171. perror("mprotect .text: PROT_READ | PROT_EXEC");
  172. return false;
  173. }
  174. }
  175. }
  176. do_main_relocations();
  177. return true;
  178. }
  179. void DynamicLoader::do_main_relocations()
  180. {
  181. do_relr_relocations();
  182. Optional<DynamicLoader::CachedLookupResult> cached_result;
  183. m_dynamic_object->relocation_section().for_each_relocation([&](DynamicObject::Relocation const& relocation) {
  184. switch (do_direct_relocation(relocation, cached_result, ShouldInitializeWeak::No, ShouldCallIfuncResolver::No)) {
  185. case RelocationResult::Failed:
  186. dbgln("Loader.so: {} unresolved symbol '{}'", m_filepath, relocation.symbol().name());
  187. VERIFY_NOT_REACHED();
  188. case RelocationResult::ResolveLater:
  189. m_unresolved_relocations.append(relocation);
  190. break;
  191. case RelocationResult::CallIfuncResolver:
  192. m_direct_ifunc_relocations.append(relocation);
  193. break;
  194. case RelocationResult::Success:
  195. break;
  196. }
  197. });
  198. // If the object is position-independent, the pointer to the PLT trampoline needs to be relocated.
  199. auto fixup_trampoline_pointer = [&](DynamicObject::Relocation const& relocation) {
  200. VERIFY(static_cast<GenericDynamicRelocationType>(relocation.type()) == GenericDynamicRelocationType::JUMP_SLOT);
  201. if (image().is_dynamic())
  202. *((FlatPtr*)relocation.address().as_ptr()) += m_dynamic_object->base_address().get();
  203. };
  204. m_dynamic_object->plt_relocation_section().for_each_relocation([&](DynamicObject::Relocation const& relocation) {
  205. if (static_cast<GenericDynamicRelocationType>(relocation.type()) == GenericDynamicRelocationType::IRELATIVE) {
  206. m_direct_ifunc_relocations.append(relocation);
  207. return;
  208. }
  209. if (static_cast<GenericDynamicRelocationType>(relocation.type()) == GenericDynamicRelocationType::TLSDESC) {
  210. // GNU ld for some reason puts TLSDESC relocations into .rela.plt
  211. // https://sourceware.org/bugzilla/show_bug.cgi?id=28387
  212. VERIFY(do_direct_relocation(relocation, cached_result, ShouldInitializeWeak::No, ShouldCallIfuncResolver::No) == RelocationResult::Success);
  213. return;
  214. }
  215. // FIXME: Or LD_BIND_NOW is set?
  216. if (m_dynamic_object->must_bind_now()) {
  217. switch (do_plt_relocation(relocation, ShouldCallIfuncResolver::No)) {
  218. case RelocationResult::Failed:
  219. dbgln("Loader.so: {} unresolved symbol '{}'", m_filepath, relocation.symbol().name());
  220. VERIFY_NOT_REACHED();
  221. case RelocationResult::ResolveLater:
  222. VERIFY_NOT_REACHED();
  223. case RelocationResult::CallIfuncResolver:
  224. m_plt_ifunc_relocations.append(relocation);
  225. // Set up lazy binding, in case an IFUNC resolver calls another IFUNC that hasn't been resolved yet.
  226. fixup_trampoline_pointer(relocation);
  227. break;
  228. case RelocationResult::Success:
  229. break;
  230. }
  231. } else {
  232. fixup_trampoline_pointer(relocation);
  233. }
  234. });
  235. }
  236. Result<NonnullRefPtr<DynamicObject>, DlErrorMessage> DynamicLoader::load_stage_3(unsigned flags)
  237. {
  238. do_lazy_relocations();
  239. if (flags & RTLD_LAZY) {
  240. if (m_dynamic_object->has_plt())
  241. setup_plt_trampoline();
  242. }
  243. // IFUNC resolvers can only be called after the PLT has been populated,
  244. // as they may call arbitrary functions via the PLT.
  245. for (auto const& relocation : m_plt_ifunc_relocations) {
  246. auto result = do_plt_relocation(relocation, ShouldCallIfuncResolver::Yes);
  247. VERIFY(result == RelocationResult::Success);
  248. }
  249. Optional<DynamicLoader::CachedLookupResult> cached_result;
  250. for (auto const& relocation : m_direct_ifunc_relocations) {
  251. auto result = do_direct_relocation(relocation, cached_result, ShouldInitializeWeak::No, ShouldCallIfuncResolver::Yes);
  252. VERIFY(result == RelocationResult::Success);
  253. }
  254. if (m_dynamic_object->has_text_relocations()) {
  255. // If we don't have textrels, .text has already been made executable by this point in load_stage_2.
  256. for (auto& text_segment : m_text_segments) {
  257. if (mprotect(text_segment.address().as_ptr(), text_segment.size(), PROT_READ | PROT_EXEC) < 0) {
  258. return DlErrorMessage { ByteString::formatted("mprotect .text: PROT_READ | PROT_EXEC: {}", strerror(errno)) };
  259. }
  260. }
  261. }
  262. if (m_relro_segment_size) {
  263. if (mprotect(m_relro_segment_address.as_ptr(), m_relro_segment_size, PROT_READ) < 0) {
  264. return DlErrorMessage { ByteString::formatted("mprotect .relro: PROT_READ: {}", strerror(errno)) };
  265. }
  266. #ifdef AK_OS_SERENITY
  267. if (set_mmap_name(m_relro_segment_address.as_ptr(), m_relro_segment_size, ByteString::formatted("{}: .relro", m_filepath).characters()) < 0) {
  268. return DlErrorMessage { ByteString::formatted("set_mmap_name .relro: {}", strerror(errno)) };
  269. }
  270. #endif
  271. }
  272. m_fully_relocated = true;
  273. return NonnullRefPtr<DynamicObject> { *m_dynamic_object };
  274. }
  275. void DynamicLoader::load_stage_4()
  276. {
  277. call_object_init_functions();
  278. m_fully_initialized = true;
  279. }
  280. void DynamicLoader::do_lazy_relocations()
  281. {
  282. Optional<DynamicLoader::CachedLookupResult> cached_result;
  283. for (auto const& relocation : m_unresolved_relocations) {
  284. if (auto res = do_direct_relocation(relocation, cached_result, ShouldInitializeWeak::Yes, ShouldCallIfuncResolver::Yes); res != RelocationResult::Success) {
  285. dbgln("Loader.so: {} unresolved symbol '{}'", m_filepath, relocation.symbol().name());
  286. VERIFY_NOT_REACHED();
  287. }
  288. }
  289. }
  290. void DynamicLoader::load_program_headers()
  291. {
  292. FlatPtr ph_load_start = SIZE_MAX;
  293. FlatPtr ph_load_end = 0;
  294. // We walk the program header list once to find the requested address ranges of the program.
  295. // We don't fill in the list of regions yet to keep malloc memory blocks from interfering with our reservation.
  296. image().for_each_program_header([&](Image::ProgramHeader const& program_header) {
  297. if (program_header.type() != PT_LOAD)
  298. return;
  299. FlatPtr section_start = program_header.vaddr().get();
  300. FlatPtr section_end = section_start + program_header.size_in_memory();
  301. if (ph_load_start > section_start)
  302. ph_load_start = section_start;
  303. if (ph_load_end < section_end)
  304. ph_load_end = section_end;
  305. });
  306. void* requested_load_address = image().is_dynamic() ? nullptr : reinterpret_cast<void*>(ph_load_start);
  307. int reservation_mmap_flags = MAP_ANON | MAP_PRIVATE | MAP_NORESERVE;
  308. if (image().is_dynamic())
  309. reservation_mmap_flags |= MAP_RANDOMIZED;
  310. #ifdef MAP_FIXED_NOREPLACE
  311. else
  312. reservation_mmap_flags |= MAP_FIXED_NOREPLACE;
  313. #endif
  314. // First, we make a dummy reservation mapping, in order to allocate enough VM
  315. // to hold all regions contiguously in the address space.
  316. FlatPtr ph_load_base = ph_load_start & ~(FlatPtr)0xfffu;
  317. ph_load_end = round_up_to_power_of_two(ph_load_end, PAGE_SIZE);
  318. size_t total_mapping_size = ph_load_end - ph_load_base;
  319. // Before we make our reservation, unmap our existing mapped ELF image that we used for reading header information.
  320. // This leaves our pointers dangling momentarily, but it reduces the chance that we will conflict with ourselves.
  321. if (munmap(m_file_data, m_file_size) < 0) {
  322. perror("munmap old mapping");
  323. VERIFY_NOT_REACHED();
  324. }
  325. m_elf_image = nullptr;
  326. m_file_data = nullptr;
  327. auto* reservation = mmap(requested_load_address, total_mapping_size, PROT_NONE, reservation_mmap_flags, 0, 0);
  328. if (reservation == MAP_FAILED) {
  329. perror("mmap reservation");
  330. VERIFY_NOT_REACHED();
  331. }
  332. // Now that we can't accidentally block our requested space, re-map our ELF image.
  333. ByteString file_mmap_name = ByteString::formatted("ELF_DYN: {}", m_filepath);
  334. auto* data = mmap_with_name(nullptr, m_file_size, PROT_READ, MAP_SHARED, m_image_fd, 0, file_mmap_name.characters());
  335. if (data == MAP_FAILED) {
  336. perror("mmap new mapping");
  337. VERIFY_NOT_REACHED();
  338. }
  339. m_file_data = data;
  340. m_elf_image = adopt_own(*new ELF::Image((u8*)m_file_data, m_file_size));
  341. VERIFY(requested_load_address == nullptr || reservation == requested_load_address);
  342. m_base_address = VirtualAddress { reservation };
  343. // Most binaries have four loadable regions, three of which are mapped
  344. // (symbol tables/relocation information, executable instructions, read-only data)
  345. // and one of which is copied (modifiable data).
  346. // These are allocated in-line to cut down on the malloc calls.
  347. Vector<ProgramHeaderRegion, 3> map_regions;
  348. Vector<ProgramHeaderRegion, 1> copy_regions;
  349. Optional<ProgramHeaderRegion> relro_region;
  350. VirtualAddress dynamic_region_desired_vaddr;
  351. image().for_each_program_header([&](Image::ProgramHeader const& program_header) {
  352. ProgramHeaderRegion region {};
  353. region.set_program_header(program_header.raw_header());
  354. if (region.is_tls_template()) {
  355. // Skip, this is handled in DynamicLoader::copy_initial_tls_data_into.
  356. } else if (region.is_load()) {
  357. if (region.size_in_memory() == 0)
  358. return;
  359. if (region.is_writable()) {
  360. copy_regions.append(region);
  361. } else {
  362. map_regions.append(region);
  363. }
  364. } else if (region.is_dynamic()) {
  365. dynamic_region_desired_vaddr = region.desired_load_address();
  366. } else if (region.is_relro()) {
  367. VERIFY(!relro_region.has_value());
  368. relro_region = region;
  369. }
  370. });
  371. VERIFY(!map_regions.is_empty() || !copy_regions.is_empty());
  372. auto compare_load_address = [](ProgramHeaderRegion& a, ProgramHeaderRegion& b) {
  373. return a.desired_load_address().as_ptr() < b.desired_load_address().as_ptr();
  374. };
  375. quick_sort(map_regions, compare_load_address);
  376. quick_sort(copy_regions, compare_load_address);
  377. // Pre-allocate any malloc memory needed before unmapping the reservation.
  378. // We don't want any future malloc to accidentally mmap a reserved address!
  379. ByteString text_segment_name = ByteString::formatted("{}: .text", m_filepath);
  380. ByteString rodata_segment_name = ByteString::formatted("{}: .rodata", m_filepath);
  381. ByteString data_segment_name = ByteString::formatted("{}: .data", m_filepath);
  382. m_text_segments.ensure_capacity(map_regions.size());
  383. // Finally, we unmap the reservation.
  384. if (munmap(reservation, total_mapping_size) < 0) {
  385. perror("munmap reservation");
  386. VERIFY_NOT_REACHED();
  387. }
  388. // WARNING: Allocating after this point has the possibility of malloc stealing our reserved
  389. // virtual memory addresses. Be careful not to malloc below!
  390. // Process regions in order: .text, .data, .tls
  391. for (auto& region : map_regions) {
  392. FlatPtr ph_desired_base = region.desired_load_address().get();
  393. FlatPtr ph_base = region.desired_load_address().page_base().get();
  394. FlatPtr ph_end = ph_base + round_up_to_power_of_two(region.size_in_memory() + region.desired_load_address().get() - ph_base, PAGE_SIZE);
  395. char const* const segment_name = region.is_executable() ? text_segment_name.characters() : rodata_segment_name.characters();
  396. // Now we can map the text segment at the reserved address.
  397. auto* segment_base = (u8*)mmap_with_name(
  398. (u8*)reservation + ph_base - ph_load_base,
  399. ph_desired_base - ph_base + region.size_in_image(),
  400. PROT_READ,
  401. MAP_SHARED | MAP_FIXED,
  402. m_image_fd,
  403. VirtualAddress { region.offset() }.page_base().get(),
  404. segment_name);
  405. if (segment_base == MAP_FAILED) {
  406. perror("mmap non-writable");
  407. VERIFY_NOT_REACHED();
  408. }
  409. // NOTE: Capacity ensured above the line of no malloc above
  410. if (region.is_executable())
  411. m_text_segments.unchecked_append({ VirtualAddress { segment_base }, ph_end - ph_base });
  412. }
  413. VERIFY(requested_load_address == nullptr || requested_load_address == reservation);
  414. if (relro_region.has_value()) {
  415. m_relro_segment_size = relro_region->size_in_memory();
  416. m_relro_segment_address = VirtualAddress { (u8*)reservation + relro_region->desired_load_address().get() - ph_load_base };
  417. }
  418. if (image().is_dynamic())
  419. m_dynamic_section_address = VirtualAddress { (u8*)reservation + dynamic_region_desired_vaddr.get() - ph_load_base };
  420. else
  421. m_dynamic_section_address = dynamic_region_desired_vaddr;
  422. for (auto& region : copy_regions) {
  423. FlatPtr ph_data_base = region.desired_load_address().page_base().get();
  424. FlatPtr ph_data_end = ph_data_base + round_up_to_power_of_two(region.size_in_memory() + region.desired_load_address().get() - ph_data_base, PAGE_SIZE);
  425. auto* data_segment_address = (u8*)reservation + ph_data_base - ph_load_base;
  426. size_t data_segment_size = ph_data_end - ph_data_base;
  427. // Finally, we make an anonymous mapping for the data segment. Contents are then copied from the file.
  428. auto* data_segment = (u8*)mmap_with_name(
  429. data_segment_address,
  430. data_segment_size,
  431. PROT_READ | PROT_WRITE,
  432. MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED,
  433. 0,
  434. 0,
  435. data_segment_name.characters());
  436. if (MAP_FAILED == data_segment) {
  437. perror("mmap writable");
  438. VERIFY_NOT_REACHED();
  439. }
  440. VirtualAddress data_segment_start;
  441. if (image().is_dynamic())
  442. data_segment_start = VirtualAddress { (u8*)reservation + region.desired_load_address().get() };
  443. else
  444. data_segment_start = region.desired_load_address();
  445. VERIFY(data_segment_start.as_ptr() + region.size_in_memory() <= data_segment + data_segment_size);
  446. memcpy(data_segment_start.as_ptr(), (u8*)m_file_data + region.offset(), region.size_in_image());
  447. }
  448. }
  449. DynamicLoader::RelocationResult DynamicLoader::do_direct_relocation(DynamicObject::Relocation const& relocation,
  450. Optional<DynamicLoader::CachedLookupResult>& cached_result,
  451. [[maybe_unused]] ShouldInitializeWeak should_initialize_weak,
  452. ShouldCallIfuncResolver should_call_ifunc_resolver)
  453. {
  454. FlatPtr* patch_ptr = nullptr;
  455. if (is_dynamic())
  456. patch_ptr = (FlatPtr*)(m_dynamic_object->base_address().as_ptr() + relocation.offset());
  457. else
  458. patch_ptr = (FlatPtr*)(FlatPtr)relocation.offset();
  459. auto call_ifunc_resolver = [](VirtualAddress address) {
  460. return VirtualAddress { reinterpret_cast<DynamicObject::IfuncResolver>(address.get())() };
  461. };
  462. auto lookup_symbol = [&](DynamicObject::Symbol const& symbol) {
  463. // The static linker sorts relocations by the referenced symbol. Especially when vtables
  464. // in large inheritance hierarchies are involved, there might be tens of references to
  465. // the same symbol. We can avoid redundant lookups by keeping track of the previous result.
  466. if (!cached_result.has_value() || !cached_result.value().symbol.definitely_equals(symbol))
  467. cached_result = DynamicLoader::CachedLookupResult { symbol, DynamicLoader::lookup_symbol(symbol) };
  468. return cached_result.value().result;
  469. };
  470. struct ResolvedTLSSymbol {
  471. DynamicObject const& dynamic_object;
  472. FlatPtr value;
  473. };
  474. auto resolve_tls_symbol = [&](DynamicObject::Relocation const& relocation) -> Optional<ResolvedTLSSymbol> {
  475. if (relocation.symbol_index() == 0)
  476. return ResolvedTLSSymbol { relocation.dynamic_object(), 0 };
  477. auto res = lookup_symbol(relocation.symbol());
  478. if (!res.has_value())
  479. return {};
  480. VERIFY(relocation.symbol().type() != STT_GNU_IFUNC);
  481. VERIFY(res.value().dynamic_object != nullptr);
  482. return ResolvedTLSSymbol { *res.value().dynamic_object, res.value().value };
  483. };
  484. using enum GenericDynamicRelocationType;
  485. switch (static_cast<GenericDynamicRelocationType>(relocation.type())) {
  486. case NONE:
  487. // Apparently most loaders will just skip these?
  488. // Seems if the 'link editor' generates one something is funky with your code
  489. break;
  490. case ABSOLUTE: {
  491. auto symbol = relocation.symbol();
  492. auto res = lookup_symbol(symbol);
  493. if (!res.has_value()) {
  494. if (symbol.bind() == STB_WEAK)
  495. return RelocationResult::ResolveLater;
  496. dbgln("ERROR: symbol not found: {}.", symbol.name());
  497. return RelocationResult::Failed;
  498. }
  499. if (res.value().type == STT_GNU_IFUNC && should_call_ifunc_resolver == ShouldCallIfuncResolver::No)
  500. return RelocationResult::CallIfuncResolver;
  501. auto symbol_address = res.value().address;
  502. if (relocation.addend_used())
  503. *patch_ptr = symbol_address.get() + relocation.addend();
  504. else
  505. *patch_ptr += symbol_address.get();
  506. if (res.value().type == STT_GNU_IFUNC)
  507. *patch_ptr = call_ifunc_resolver(VirtualAddress { *patch_ptr }).get();
  508. break;
  509. }
  510. #if !ARCH(RISCV64)
  511. case GLOB_DAT: {
  512. auto symbol = relocation.symbol();
  513. auto res = lookup_symbol(symbol);
  514. VirtualAddress symbol_location;
  515. if (!res.has_value()) {
  516. if (symbol.bind() == STB_WEAK) {
  517. if (should_initialize_weak == ShouldInitializeWeak::No)
  518. return RelocationResult::ResolveLater;
  519. } else {
  520. // Symbol not found
  521. return RelocationResult::Failed;
  522. }
  523. symbol_location = VirtualAddress { (FlatPtr)0 };
  524. } else {
  525. symbol_location = res.value().address;
  526. if (res.value().type == STT_GNU_IFUNC) {
  527. if (should_call_ifunc_resolver == ShouldCallIfuncResolver::No)
  528. return RelocationResult::CallIfuncResolver;
  529. if (res.value().dynamic_object != nullptr && res.value().dynamic_object->has_text_relocations()) {
  530. dbgln("\033[31mError:\033[0m Refusing to call IFUNC resolver defined in an object with text relocations.");
  531. return RelocationResult::Failed;
  532. }
  533. symbol_location = call_ifunc_resolver(symbol_location);
  534. }
  535. }
  536. VERIFY(symbol_location != m_dynamic_object->base_address());
  537. *patch_ptr = symbol_location.get();
  538. break;
  539. }
  540. #endif
  541. case RELATIVE: {
  542. if (!image().is_dynamic())
  543. break;
  544. // FIXME: According to the spec, R_386_relative ones must be done first.
  545. // We could explicitly do them first using m_number_of_relocations from DT_RELCOUNT
  546. // However, our compiler is nice enough to put them at the front of the relocations for us :)
  547. if (relocation.addend_used())
  548. *patch_ptr = m_dynamic_object->base_address().offset(relocation.addend()).get();
  549. else
  550. *patch_ptr += m_dynamic_object->base_address().get();
  551. break;
  552. }
  553. case TLS_TPREL: {
  554. auto maybe_resolution = resolve_tls_symbol(relocation);
  555. if (!maybe_resolution.has_value())
  556. break;
  557. auto [dynamic_object_of_symbol, symbol_value] = maybe_resolution.value();
  558. size_t addend = relocation.addend_used() ? relocation.addend() : *patch_ptr;
  559. *patch_ptr = addend + dynamic_object_of_symbol.tls_offset().value() + symbol_value;
  560. // At offset 0 there's the thread's ThreadSpecificData structure, we don't want to collide with it.
  561. VERIFY(static_cast<ssize_t>(*patch_ptr) < 0);
  562. break;
  563. }
  564. case TLS_DTPMOD: {
  565. auto maybe_resolution = resolve_tls_symbol(relocation);
  566. if (!maybe_resolution.has_value())
  567. break;
  568. // We repurpose the module index to store the TLS block's TP offset. This is fine
  569. // because we currently only support a single static TLS block.
  570. *patch_ptr = maybe_resolution->dynamic_object.tls_offset().value();
  571. break;
  572. }
  573. case TLS_DTPREL: {
  574. auto maybe_resolution = resolve_tls_symbol(relocation);
  575. if (!maybe_resolution.has_value())
  576. break;
  577. size_t addend = relocation.addend_used() ? relocation.addend() : *patch_ptr;
  578. *patch_ptr = addend + maybe_resolution->value;
  579. break;
  580. }
  581. #ifdef HAS_TLSDESC_SUPPORT
  582. case TLSDESC: {
  583. auto maybe_resolution = resolve_tls_symbol(relocation);
  584. if (!maybe_resolution.has_value())
  585. break;
  586. auto [dynamic_object_of_symbol, symbol_value] = maybe_resolution.value();
  587. size_t addend = relocation.addend_used() ? relocation.addend() : *patch_ptr;
  588. patch_ptr[0] = (FlatPtr)__tlsdesc_static;
  589. patch_ptr[1] = addend + dynamic_object_of_symbol.tls_offset().value() + symbol_value;
  590. break;
  591. }
  592. #endif
  593. case IRELATIVE: {
  594. if (should_call_ifunc_resolver == ShouldCallIfuncResolver::No)
  595. return RelocationResult::CallIfuncResolver;
  596. VirtualAddress resolver;
  597. if (relocation.addend_used())
  598. resolver = m_dynamic_object->base_address().offset(relocation.addend());
  599. else
  600. resolver = m_dynamic_object->base_address().offset(*patch_ptr);
  601. if (m_dynamic_object->has_text_relocations()) {
  602. dbgln("\033[31mError:\033[0m Refusing to call IFUNC resolver defined in an object with text relocations.");
  603. return RelocationResult::Failed;
  604. }
  605. *patch_ptr = call_ifunc_resolver(resolver).get();
  606. break;
  607. }
  608. case JUMP_SLOT:
  609. VERIFY_NOT_REACHED(); // PLT relocations are handled by do_plt_relocation.
  610. default:
  611. // Raise the alarm! Someone needs to implement this relocation type
  612. dbgln("Found a new exciting relocation type {}", relocation.type());
  613. VERIFY_NOT_REACHED();
  614. }
  615. return RelocationResult::Success;
  616. }
  617. DynamicLoader::RelocationResult DynamicLoader::do_plt_relocation(DynamicObject::Relocation const& relocation, ShouldCallIfuncResolver should_call_ifunc_resolver)
  618. {
  619. VERIFY(static_cast<GenericDynamicRelocationType>(relocation.type()) == GenericDynamicRelocationType::JUMP_SLOT);
  620. auto symbol = relocation.symbol();
  621. auto* relocation_address = (FlatPtr*)relocation.address().as_ptr();
  622. VirtualAddress symbol_location {};
  623. if (auto result = lookup_symbol(symbol); result.has_value()) {
  624. auto address = result.value().address;
  625. if (result.value().type == STT_GNU_IFUNC) {
  626. if (should_call_ifunc_resolver == ShouldCallIfuncResolver::No)
  627. return RelocationResult::CallIfuncResolver;
  628. symbol_location = VirtualAddress { reinterpret_cast<DynamicObject::IfuncResolver>(address.get())() };
  629. } else {
  630. symbol_location = address;
  631. }
  632. } else if (symbol.bind() != STB_WEAK) {
  633. return RelocationResult::Failed;
  634. }
  635. dbgln_if(DYNAMIC_LOAD_DEBUG, "DynamicLoader: Jump slot relocation: putting {} ({}) into PLT at {}", symbol.name(), symbol_location, (void*)relocation_address);
  636. *relocation_address = symbol_location.get();
  637. return RelocationResult::Success;
  638. }
  639. void DynamicLoader::do_relr_relocations()
  640. {
  641. auto base_address = m_dynamic_object->base_address().get();
  642. m_dynamic_object->for_each_relr_relocation([base_address](FlatPtr address) {
  643. *(FlatPtr*)address += base_address;
  644. });
  645. }
  646. void DynamicLoader::copy_initial_tls_data_into(ByteBuffer& buffer) const
  647. {
  648. image().for_each_program_header([this, &buffer](ELF::Image::ProgramHeader program_header) {
  649. if (program_header.type() != PT_TLS)
  650. return IterationDecision::Continue;
  651. // Note: The "size in image" is only concerned with initialized data. Uninitialized data (.tbss) is
  652. // only included in the "size in memory" metric, and is expected to not be touched or read from, as
  653. // it is not present in the image and zeroed out in-memory. We will still check that the buffer has
  654. // space for both the initialized and the uninitialized data.
  655. // Note: The m_tls_offset here is (of course) negative.
  656. // TODO: Is the initialized data always in the beginning of the TLS segment, or should we walk the
  657. // sections to figure that out?
  658. size_t tls_start_in_buffer = buffer.size() + m_tls_offset;
  659. VERIFY(program_header.size_in_image() <= program_header.size_in_memory());
  660. VERIFY(program_header.size_in_memory() <= m_tls_size_of_current_object);
  661. VERIFY(tls_start_in_buffer + program_header.size_in_memory() <= buffer.size());
  662. memcpy(buffer.data() + tls_start_in_buffer, static_cast<const u8*>(m_file_data) + program_header.offset(), program_header.size_in_image());
  663. return IterationDecision::Break;
  664. });
  665. }
  666. // Defined in <arch>/plt_trampoline.S
  667. extern "C" void _plt_trampoline(void) __attribute__((visibility("hidden")));
  668. void DynamicLoader::setup_plt_trampoline()
  669. {
  670. VERIFY(m_dynamic_object);
  671. VERIFY(m_dynamic_object->has_plt());
  672. VirtualAddress got_address = m_dynamic_object->plt_got_base_address();
  673. auto* got_ptr = (FlatPtr*)got_address.as_ptr();
  674. #if ARCH(AARCH64) || ARCH(X86_64)
  675. got_ptr[1] = (FlatPtr)m_dynamic_object.ptr();
  676. got_ptr[2] = (FlatPtr)&_plt_trampoline;
  677. #elif ARCH(RISCV64)
  678. got_ptr[0] = (FlatPtr)&_plt_trampoline;
  679. got_ptr[1] = (FlatPtr)m_dynamic_object.ptr();
  680. #else
  681. # error Unknown architecture
  682. #endif
  683. }
  684. // Called from our ASM routine _plt_trampoline.
  685. extern "C" FlatPtr _fixup_plt_entry(DynamicObject* object, u32 relocation_offset)
  686. {
  687. auto const& relocation = object->plt_relocation_section().relocation_at_offset(relocation_offset);
  688. auto result = DynamicLoader::do_plt_relocation(relocation, ShouldCallIfuncResolver::Yes);
  689. if (result != DynamicLoader::RelocationResult::Success) {
  690. dbgln("Loader.so: {} unresolved symbol '{}'", object->filepath(), relocation.symbol().name());
  691. VERIFY_NOT_REACHED();
  692. }
  693. return *reinterpret_cast<FlatPtr*>(relocation.address().as_ptr());
  694. }
  695. void DynamicLoader::call_object_init_functions()
  696. {
  697. typedef void (*InitFunc)();
  698. if (m_dynamic_object->has_init_section()) {
  699. auto init_function = m_dynamic_object->init_section_function();
  700. (init_function)();
  701. }
  702. if (m_dynamic_object->has_init_array_section()) {
  703. auto init_array_section = m_dynamic_object->init_array_section();
  704. InitFunc* init_begin = (InitFunc*)(init_array_section.address().as_ptr());
  705. InitFunc* init_end = init_begin + init_array_section.entry_count();
  706. while (init_begin != init_end) {
  707. // Android sources claim that these can be -1, to be ignored.
  708. // 0 definitely shows up. Apparently 0/-1 are valid? Confusing.
  709. if (!*init_begin || ((FlatPtr)*init_begin == (FlatPtr)-1))
  710. continue;
  711. (*init_begin)();
  712. ++init_begin;
  713. }
  714. }
  715. }
  716. Optional<DynamicObject::SymbolLookupResult> DynamicLoader::lookup_symbol(const ELF::DynamicObject::Symbol& symbol)
  717. {
  718. if (symbol.is_undefined() || symbol.bind() == STB_WEAK)
  719. return DynamicLinker::lookup_global_symbol(symbol.name());
  720. return DynamicObject::SymbolLookupResult { symbol.value(), symbol.size(), symbol.address(), symbol.bind(), symbol.type(), &symbol.object() };
  721. }
  722. } // end namespace ELF