diff --git a/benchmarks/Android.bp b/benchmarks/Android.bp index 5dfc38f5cd..61e1f41711 100644 --- a/benchmarks/Android.bp +++ b/benchmarks/Android.bp @@ -33,7 +33,7 @@ license { cc_defaults { name: "bionic-benchmarks-defaults", cflags: [ - "-O2", + "-O3", "-fno-builtin", "-Wall", "-Wextra", @@ -86,6 +86,7 @@ cc_defaults { "-Wextra", "-Werror", "-Wunused", + "-O3", ], } @@ -156,3 +157,23 @@ cc_test { ], data: ["test_suites/*"], } + +cc_binary { + name: "malloc-rss-benchmark", + srcs: [ + "malloc_rss_benchmark.cpp", + ], + + shared_libs: [ + "libbase", + ], + + target: { + android: { + static_libs: [ + "libmeminfo", + "libprocinfo", + ], + }, + }, +} diff --git a/benchmarks/NOTICE b/benchmarks/NOTICE index f720e2331d..e46a6242f3 100644 --- a/benchmarks/NOTICE +++ b/benchmarks/NOTICE @@ -178,3 +178,31 @@ SUCH DAMAGE. ------------------------------------------------------------------- +Copyright (C) 2022 The Android Open Source Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +------------------------------------------------------------------- + diff --git a/benchmarks/malloc_benchmark.cpp b/benchmarks/malloc_benchmark.cpp index 18ba52386a..1c7329c695 100644 --- a/benchmarks/malloc_benchmark.cpp +++ b/benchmarks/malloc_benchmark.cpp @@ -40,7 +40,7 @@ static void BM_mallopt_purge(benchmark::State& state) { static size_t sizes[] = {8, 16, 32, 64, 128, 1024, 4096, 16384, 65536, 131072, 1048576}; static int pagesize = getpagesize(); mallopt(M_DECAY_TIME, 1); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); for (auto _ : state) { state.PauseTiming(); std::vector ptrs; @@ -63,7 +63,7 @@ static void BM_mallopt_purge(benchmark::State& state) { ptrs.clear(); state.ResumeTiming(); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); } mallopt(M_DECAY_TIME, 0); } diff --git a/benchmarks/malloc_map_benchmark.cpp b/benchmarks/malloc_map_benchmark.cpp index ba4d62c0bf..5757325392 100644 --- a/benchmarks/malloc_map_benchmark.cpp +++ b/benchmarks/malloc_map_benchmark.cpp @@ -69,7 +69,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) { for (auto _ : state) { #if defined(__BIONIC__) state.PauseTiming(); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); uint64_t rss_bytes_before = 0; Gather(&rss_bytes_before); state.ResumeTiming(); @@ -80,7 +80,7 @@ static void MapBenchmark(benchmark::State& state, size_t num_elements) { } #if defined(__BIONIC__) state.PauseTiming(); - mallopt(M_PURGE, 0); + mallopt(M_PURGE_ALL, 0); Gather(&rss_bytes); // Try and record only the memory used in the map. rss_bytes -= rss_bytes_before; diff --git a/benchmarks/malloc_rss_benchmark.cpp b/benchmarks/malloc_rss_benchmark.cpp new file mode 100644 index 0000000000..4b34e72f7c --- /dev/null +++ b/benchmarks/malloc_rss_benchmark.cpp @@ -0,0 +1,165 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#if defined(__BIONIC__) +#include +#include +#include +#endif + +constexpr size_t kMaxThreads = 8; +// The max number of bytes that can be allocated by a thread. Note that each +// allocator may have its own limitation on each size allocation. For example, +// Scudo has a 256 MB limit for each size-class in the primary allocator. The +// amount of memory allocated should not exceed the limit in each allocator. +constexpr size_t kMaxBytes = 1 << 24; +constexpr size_t kMaxLen = kMaxBytes; +void* MemPool[kMaxThreads][kMaxLen]; + +void dirtyMem(void* ptr, size_t bytes) { + memset(ptr, 1U, bytes); +} + +void ThreadTask(int id, size_t allocSize) { + // In the following, we will first allocate blocks with kMaxBytes of memory + // and release all of them in random order. In the end, we will do another + // round of allocations until it reaches 1/10 kMaxBytes. + + // Total number of blocks + const size_t maxCounts = kMaxBytes / allocSize; + // The number of blocks in the end + const size_t finalCounts = maxCounts / 10; + + for (size_t i = 0; i < maxCounts; ++i) { + MemPool[id][i] = malloc(allocSize); + if (MemPool[id][i] == 0) { + std::cout << "Allocation failure." + "Please consider reducing the number of threads" + << std::endl; + exit(1); + } + dirtyMem(MemPool[id][i], allocSize); + } + + // Each allocator may apply different strategies to manage the free blocks and + // each strategy may have different impacts on future memory usage. For + // example, managing free blocks in simple FIFO list may have its memory usage + // highly correlated with the blocks releasing pattern. Therefore, release the + // blocks in random order to observe the impact of free blocks handling. + unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); + std::shuffle(MemPool[id], MemPool[id] + maxCounts, std::default_random_engine(seed)); + for (size_t i = 0; i < maxCounts; ++i) { + free(MemPool[id][i]); + MemPool[id][i] = nullptr; + } + + for (size_t i = 0; i < finalCounts; ++i) { + MemPool[id][i] = malloc(allocSize); + dirtyMem(MemPool[id][i], allocSize); + } +} + +void StressSizeClass(size_t numThreads, size_t allocSize) { + // We would like to see the minimum memory usage under aggressive page + // releasing. + mallopt(M_DECAY_TIME, 0); + + std::thread* threads[kMaxThreads]; + for (size_t i = 0; i < numThreads; ++i) threads[i] = new std::thread(ThreadTask, i, allocSize); + + for (size_t i = 0; i < numThreads; ++i) { + threads[i]->join(); + delete threads[i]; + } + + // Do an explicit purge to ensure we will be more likely to get the actual + // in-use memory. + mallopt(M_PURGE_ALL, 0); + + android::meminfo::ProcMemInfo proc_mem(getpid()); + const std::vector& maps = proc_mem.MapsWithoutUsageStats(); + uint64_t rss_bytes = 0; + uint64_t vss_bytes = 0; + + for (auto& vma : maps) { + if (vma.name == "[anon:libc_malloc]" || android::base::StartsWith(vma.name, "[anon:scudo:") || + android::base::StartsWith(vma.name, "[anon:GWP-ASan")) { + android::meminfo::Vma update_vma(vma); + if (!proc_mem.FillInVmaStats(update_vma)) { + std::cout << "Failed to parse VMA" << std::endl; + exit(1); + } + rss_bytes += update_vma.usage.rss; + vss_bytes += update_vma.usage.vss; + } + } + + std::cout << "RSS: " << rss_bytes / (1024.0 * 1024.0) << " MB" << std::endl; + std::cout << "VSS: " << vss_bytes / (1024.0 * 1024.0) << " MB" << std::endl; + + for (size_t i = 0; i < numThreads; ++i) { + for (size_t j = 0; j < kMaxLen; ++j) free(MemPool[i][j]); + } +} + +int main(int argc, char* argv[]) { + if (argc != 3) { + std::cerr << "usage: " << argv[0] << " $NUM_THREADS $ALLOC_SIZE" << std::endl; + return 1; + } + + size_t numThreads = atoi(argv[1]); + size_t allocSize = atoi(argv[2]); + + if (numThreads == 0 || allocSize == 0) { + std::cerr << "Please provide valid $NUM_THREADS and $ALLOC_SIZE" << std::endl; + return 1; + } + + if (numThreads > kMaxThreads) { + std::cerr << "The max number of threads is " << kMaxThreads << std::endl; + return 1; + } + + StressSizeClass(numThreads, allocSize); + + return 0; +} diff --git a/libc/Android.bp b/libc/Android.bp index 97146aa6f8..4fdd46cfe1 100644 --- a/libc/Android.bp +++ b/libc/Android.bp @@ -55,6 +55,7 @@ libc_common_flags = [ "-Wno-deprecated-declarations", "-Wno-gcc-compat", "-Wframe-larger-than=2048", + "-O3", // Try to catch typical 32-bit assumptions that break with 64-bit pointers. "-Werror=pointer-to-int-cast", @@ -126,9 +127,9 @@ cc_defaults { malloc_pattern_fill_contents: { cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"], }, - malloc_not_svelte: { - cflags: ["-DUSE_SCUDO"], - }, + //malloc_not_svelte: { + //cflags: ["-DUSE_SCUDO"], + //}, }, lto: { @@ -137,14 +138,14 @@ cc_defaults { } libc_scudo_product_variables = { - malloc_not_svelte: { - cflags: ["-DUSE_SCUDO"], - whole_static_libs: ["libscudo"], - exclude_static_libs: [ - "libjemalloc5", - "libc_jemalloc_wrapper", - ], - }, + //malloc_not_svelte: { + //cflags: ["-DUSE_SCUDO"], + //whole_static_libs: ["libscudo"], + //exclude_static_libs: [ + //"libjemalloc5", + //"libc_jemalloc_wrapper", + //], + //}, } // Defaults for native allocator libs/includes to make it @@ -161,7 +162,6 @@ cc_defaults { "libc_jemalloc_wrapper", ], header_libs: ["gwp_asan_headers"], - product_variables: libc_scudo_product_variables, } // Functions not implemented by jemalloc directly, or that need to @@ -272,6 +272,7 @@ cc_library_static { "-DWILDABBR=\"\"", "-DNO_RUN_TIME_WARNINGS_ABOUT_YEAR_2000_PROBLEMS_THANK_YOU", "-Dlint", + "-O3", ], local_include_dirs: ["tzcode/"], @@ -381,6 +382,7 @@ cc_library_static { "-Wno-sign-compare", "-Wno-unused-parameter", "-include freebsd-compat.h", + "-O3", ], local_include_dirs: [ @@ -400,6 +402,7 @@ cc_library_static { "-Wno-sign-compare", "-include freebsd-compat.h", "-Wframe-larger-than=66000", + "-O3", ], local_include_dirs: [ @@ -454,6 +457,7 @@ cc_library_static { "-Wno-unused-parameter", "-DPOSIX_MISTAKE", "-include netbsd-compat.h", + "-O3", ], local_include_dirs: [ @@ -586,6 +590,7 @@ cc_library_static { "-Wno-sign-compare", "-Wno-unused-parameter", "-include openbsd-compat.h", + "-O3", ], local_include_dirs: [ @@ -610,6 +615,7 @@ cc_library_static { "-include openbsd-compat.h", "-Wno-sign-compare", "-Wframe-larger-than=5000", + "-O3", ], local_include_dirs: [ @@ -698,6 +704,7 @@ cc_library_static { "-Wno-sign-compare", "-Wno-unused-parameter", "-include openbsd-compat.h", + "-O3", ], local_include_dirs: [ @@ -746,6 +753,7 @@ cc_library_static { cflags: [ "-Wno-sign-compare", "-include openbsd-compat.h", + "-O3", ], local_include_dirs: [ @@ -772,6 +780,7 @@ cc_library_static { cflags: [ "-U_FORTIFY_SOURCE", "-D__BIONIC_DECLARE_FORTIFY_HELPERS", + "-O3", ], arch: { @@ -1253,7 +1262,10 @@ cc_library_static { // ======================================================== cc_library_static { - defaults: ["libc_defaults"], + defaults: [ + "libc_defaults", + "target_alternative_futex_waiters_defaults" + ], srcs: [ "bionic/bionic_elf_tls.cpp", "bionic/pthread_atfork.cpp", @@ -1427,6 +1439,7 @@ cc_library_static { cflags: [ "-fvisibility=hidden", "-DLIBC_STATIC", + "-O3", ], whole_static_libs: [ @@ -1998,6 +2011,7 @@ cc_library { name: "libstdc++", static_ndk_lib: true, static_libs: ["libasync_safe"], + vendor_available: true, static: { system_shared_libs: [], @@ -2089,6 +2103,7 @@ cc_defaults { "-Wno-gcc-compat", "-Wall", "-Werror", + "-O3", ], sanitize: { never: true, @@ -2227,7 +2242,9 @@ cc_library_static { } }, sdk_version: "minimum", - + lto: { + thin: true, + }, defaults: ["crt_and_memtag_defaults"], } @@ -2239,7 +2256,9 @@ cc_library_static { } }, sdk_version: "minimum", - + lto: { + thin: true, + }, defaults: ["crt_and_memtag_defaults"], } diff --git a/libc/async_safe/Android.bp b/libc/async_safe/Android.bp index 531317d152..fe89687caa 100644 --- a/libc/async_safe/Android.bp +++ b/libc/async_safe/Android.bp @@ -39,6 +39,9 @@ cc_library_static { "com.android.virt", ], min_sdk_version: "apex_inherit", + lto: { + full: true, + }, } cc_library_headers { diff --git a/libc/async_safe/async_safe_log.cpp b/libc/async_safe/async_safe_log.cpp index 2380e686d9..420560f7b0 100644 --- a/libc/async_safe/async_safe_log.cpp +++ b/libc/async_safe/async_safe_log.cpp @@ -254,7 +254,7 @@ static void out_vformat(Out& o, const char* format, va_list args) { bool alternate = false; size_t bytelen = sizeof(int); int slen; - char buffer[32]; /* temporary buffer used to format numbers */ + char buffer[64]; // temporary buffer used to format numbers/format errno string char c; @@ -359,8 +359,7 @@ static void out_vformat(Out& o, const char* format, va_list args) { buffer[1] = 'x'; format_integer(buffer + 2, sizeof(buffer) - 2, value, 'x'); } else if (c == 'm') { - char buf[256]; - str = strerror_r(errno, buf, sizeof(buf)); + strerror_r(errno, buffer, sizeof(buffer)); } else if (c == 'd' || c == 'i' || c == 'o' || c == 'u' || c == 'x' || c == 'X') { /* integers - first read value from stack */ uint64_t value; diff --git a/libc/bionic/fdsan.cpp b/libc/bionic/fdsan.cpp index 48e8674d6d..66d40d824d 100644 --- a/libc/bionic/fdsan.cpp +++ b/libc/bionic/fdsan.cpp @@ -101,7 +101,7 @@ FdEntry* FdTableImpl::at(size_t idx) { } size_t offset = idx - inline_fds; - if (local_overflow->len < offset) { + if (local_overflow->len <= offset) { return nullptr; } return &local_overflow->entries[offset]; diff --git a/libc/bionic/jemalloc_wrapper.cpp b/libc/bionic/jemalloc_wrapper.cpp index ef488eecc9..ce3f314203 100644 --- a/libc/bionic/jemalloc_wrapper.cpp +++ b/libc/bionic/jemalloc_wrapper.cpp @@ -102,7 +102,7 @@ int je_mallopt(int param, int value) { } } return 1; - } else if (param == M_PURGE) { + } else if (param == M_PURGE || param == M_PURGE_ALL) { // Only clear the current thread cache since there is no easy way to // clear the caches of other threads. // This must be done first so that cleared allocations get purged diff --git a/libc/bionic/malloc_common_dynamic.cpp b/libc/bionic/malloc_common_dynamic.cpp index 6c2f4d9415..b2ba825dc9 100644 --- a/libc/bionic/malloc_common_dynamic.cpp +++ b/libc/bionic/malloc_common_dynamic.cpp @@ -371,6 +371,7 @@ static bool InstallHooks(libc_globals* globals, const char* options, const char* extern "C" const char* __scudo_get_stack_depot_addr(); extern "C" const char* __scudo_get_region_info_addr(); extern "C" const char* __scudo_get_ring_buffer_addr(); +extern "C" size_t __scudo_get_ring_buffer_size(); // Initializes memory allocation framework once per process. static void MallocInitImpl(libc_globals* globals) { @@ -383,6 +384,7 @@ static void MallocInitImpl(libc_globals* globals) { __libc_shared_globals()->scudo_stack_depot = __scudo_get_stack_depot_addr(); __libc_shared_globals()->scudo_region_info = __scudo_get_region_info_addr(); __libc_shared_globals()->scudo_ring_buffer = __scudo_get_ring_buffer_addr(); + __libc_shared_globals()->scudo_ring_buffer_size = __scudo_get_ring_buffer_size(); #endif // Prefer malloc debug since it existed first and is a more complete diff --git a/libc/bionic/pthread_cond.cpp b/libc/bionic/pthread_cond.cpp index 793dcd9421..5e88814557 100644 --- a/libc/bionic/pthread_cond.cpp +++ b/libc/bionic/pthread_cond.cpp @@ -116,9 +116,13 @@ struct pthread_cond_internal_t { } #if defined(__LP64__) +#if defined(TARGET_ALTERNATIVE_FUTEX_WAITERS) + char __reserved[44]; +#else atomic_uint waiters; char __reserved[40]; #endif +#endif }; static_assert(sizeof(pthread_cond_t) == sizeof(pthread_cond_internal_t), @@ -143,7 +147,9 @@ int pthread_cond_init(pthread_cond_t* cond_interface, const pthread_condattr_t* atomic_init(&cond->state, init_state); #if defined(__LP64__) +#if !defined(TARGET_ALTERNATIVE_FUTEX_WAITERS) atomic_init(&cond->waiters, 0); +#endif #endif return 0; @@ -169,9 +175,11 @@ static int __pthread_cond_pulse(pthread_cond_internal_t* cond, int thread_count) // synchronization. And it doesn't help even if we use any fence here. #if defined(__LP64__) +#if !defined(TARGET_ALTERNATIVE_FUTEX_WAITERS) if (atomic_load_explicit(&cond->waiters, memory_order_relaxed) == 0) { return 0; } +#endif #endif // The increase of value should leave flags alone, even if the value can overflows. @@ -191,7 +199,9 @@ static int __pthread_cond_timedwait(pthread_cond_internal_t* cond, pthread_mutex unsigned int old_state = atomic_load_explicit(&cond->state, memory_order_relaxed); #if defined(__LP64__) +#if !defined(TARGET_ALTERNATIVE_FUTEX_WAITERS) atomic_fetch_add_explicit(&cond->waiters, 1, memory_order_relaxed); +#endif #endif pthread_mutex_unlock(mutex); @@ -199,7 +209,9 @@ static int __pthread_cond_timedwait(pthread_cond_internal_t* cond, pthread_mutex use_realtime_clock, abs_timeout_or_null); #if defined(__LP64__) +#if !defined(TARGET_ALTERNATIVE_FUTEX_WAITERS) atomic_fetch_sub_explicit(&cond->waiters, 1, memory_order_relaxed); +#endif #endif pthread_mutex_lock(mutex); diff --git a/libc/include/arpa/inet.h b/libc/include/arpa/inet.h index db054c9e1d..7716b94457 100644 --- a/libc/include/arpa/inet.h +++ b/libc/include/arpa/inet.h @@ -33,6 +33,7 @@ #include #include #include +#include __BEGIN_DECLS diff --git a/libc/include/bits/in_addr.h b/libc/include/bits/in_addr.h index 30eb04b668..3e46dad2b0 100644 --- a/libc/include/bits/in_addr.h +++ b/libc/include/bits/in_addr.h @@ -36,8 +36,7 @@ #include #include -/** An integral type representing an IPv4 address. */ -typedef uint32_t in_addr_t; +#include /** A structure representing an IPv4 address. */ struct in_addr { diff --git a/libc/include/inaddr.h b/libc/include/inaddr.h new file mode 100644 index 0000000000..524addabf6 --- /dev/null +++ b/libc/include/inaddr.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _INADDR_H_ +#define _INADDR_H_ + +#include + +typedef uint32_t in_addr_t; + +#endif diff --git a/libc/include/malloc.h b/libc/include/malloc.h index 40786fad69..c3f261398c 100644 --- a/libc/include/malloc.h +++ b/libc/include/malloc.h @@ -183,7 +183,15 @@ int malloc_info(int __must_be_zero, FILE* __fp) __INTRODUCED_IN(23); * Available since API level 28. */ #define M_PURGE (-101) - +/** + * mallopt() option to immediately purge all possible memory back to + * the kernel. This call can take longer than a normal purge since it + * examines everything. In some cases, it can take more than twice the + * time of a M_PURGE call. The value is ignored. + * + * Available since API level 34. + */ +#define M_PURGE_ALL (-300) /** * mallopt() option to tune the allocator's choice of memory tags to diff --git a/libc/malloc_debug/malloc_debug.cpp b/libc/malloc_debug/malloc_debug.cpp index 9f38946af6..7177a1e31f 100644 --- a/libc/malloc_debug/malloc_debug.cpp +++ b/libc/malloc_debug/malloc_debug.cpp @@ -399,6 +399,9 @@ void debug_get_malloc_leak_info(uint8_t** info, size_t* overall_size, size_t* in void debug_free_malloc_leak_info(uint8_t* info) { g_dispatch->free(info); + // Purge the memory that was freed since a significant amount of + // memory could have been allocated and freed. + g_dispatch->mallopt(M_PURGE_ALL, 0); } size_t debug_malloc_usable_size(void* pointer) { @@ -975,6 +978,10 @@ static void write_dump(int fd) { dprintf(fd, "%s", content.c_str()); } dprintf(fd, "END\n"); + + // Purge the memory that was allocated and freed during this operation + // since it can be large enough to expand the RSS significantly. + g_dispatch->mallopt(M_PURGE_ALL, 0); } bool debug_write_malloc_leak_info(FILE* fp) { diff --git a/libc/private/bionic_globals.h b/libc/private/bionic_globals.h index e105c18d93..aca8626119 100644 --- a/libc/private/bionic_globals.h +++ b/libc/private/bionic_globals.h @@ -110,6 +110,7 @@ struct libc_shared_globals { const char* scudo_stack_depot = nullptr; const char* scudo_region_info = nullptr; const char* scudo_ring_buffer = nullptr; + size_t scudo_ring_buffer_size = 0; HeapTaggingLevel initial_heap_tagging_level = M_HEAP_TAGGING_LEVEL_NONE; }; diff --git a/libdl/Android.bp b/libdl/Android.bp index 750a6e26f5..19e1b5280f 100644 --- a/libdl/Android.bp +++ b/libdl/Android.bp @@ -34,6 +34,7 @@ cc_library_static { "-Wextra", "-Wunused", "-Werror", + "-O3", ], // For private/CFIShadow.h. @@ -197,6 +198,7 @@ cc_library { "-Wextra", "-Wunused", "-Werror", + "-O3", ], stl: "none", diff --git a/linker/Android.bp b/linker/Android.bp index d5e7367f4f..408c158d52 100644 --- a/linker/Android.bp +++ b/linker/Android.bp @@ -43,6 +43,7 @@ cc_object { "-Wextra", "-Wno-unused", "-Werror", + "-O3", ], srcs: [ @@ -70,6 +71,9 @@ cc_object { // Configuration for the linker binary and any of its static libraries. cc_defaults { name: "linker_defaults", + defaults: [ + "shim_libs_defaults", + ], arch: { arm: { cflags: ["-D__work_around_b_24465209__"], @@ -115,7 +119,7 @@ cc_defaults { "libasync_safe", - "liblog", + "liblog_for_runtime_apex", ], // We need to access Bionic private headers in the linker. @@ -263,6 +267,8 @@ cc_defaults { "-Wl,-soname,ld-android.so", ], + cflags: ["-O3"], + // we are going to link libc++_static manually because // when stl is not set to "none" build system adds libdl // to the list of static libraries which needs to be @@ -426,6 +432,7 @@ cc_library { "-Wextra", "-Wunused", "-Werror", + "-O3", ], stl: "none", @@ -494,7 +501,7 @@ cc_test { static_libs: [ "libasync_safe", "libbase", - "liblog", + "liblog_for_runtime_apex", ], arch: { diff --git a/linker/linker.cpp b/linker/linker.cpp index c6588d2cd4..76450f87bd 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -655,6 +655,68 @@ enum walk_action_result_t : uint32_t { kWalkSkip = 2 }; +#ifdef LD_SHIM_LIBS +// g_ld_all_shim_libs maintains the references to memory as it used +// in the soinfo structures and in the g_active_shim_libs list. + +static std::vector g_ld_all_shim_libs; + +// g_active_shim_libs are all shim libs that are still eligible +// to be loaded. We must remove a shim lib from the list before +// we load the library to avoid recursive loops (load shim libA +// for libB where libA also links against libB). +static linked_list_t g_active_shim_libs; + +static void reset_g_active_shim_libs(void) { + g_active_shim_libs.clear(); + for (const auto& pair : g_ld_all_shim_libs) { + g_active_shim_libs.push_back(&pair); + } +} + +void parse_LD_SHIM_LIBS(const char* path) { + g_ld_all_shim_libs.clear(); + if (path != nullptr) { + for (const auto& pair : android::base::Split(path, ":")) { + std::vector pieces = android::base::Split(pair, "|"); + if (pieces.size() != 2) continue; + // If the path can be resolved, resolve it + char buf[PATH_MAX]; + std::string resolved_path = pieces[0]; + if (access(pieces[0].c_str(), R_OK) != 0) { + if (errno == ENOENT) { + // no need to test for non-existing path. skip. + continue; + } + // If not accessible, don't call realpath as it will just cause + // SELinux denial spam. Use the path unresolved. + } else if (realpath(pieces[0].c_str(), buf) != nullptr) { + resolved_path = buf; + } + auto desc = std::pair(resolved_path, pieces[1]); + g_ld_all_shim_libs.push_back(desc); + } + } + reset_g_active_shim_libs(); +} + +std::vector shim_matching_pairs(const char* path) { + std::vector matched_pairs; + + g_active_shim_libs.for_each([&](const ShimDescriptor* a_pair) { + if (a_pair->first == path) { + matched_pairs.push_back(a_pair); + } + }); + + g_active_shim_libs.remove_if([&](const ShimDescriptor* a_pair) { + return a_pair->first == path; + }); + + return matched_pairs; +} +#endif + // This function walks down the tree of soinfo dependencies // in breadth-first order and // * calls action(soinfo* si) for each node, and @@ -842,14 +904,14 @@ class ZipArchiveCache { ZipArchiveCache() {} ~ZipArchiveCache(); - bool get_or_open(const char* zip_path, ZipArchiveHandle* handle); + bool get_or_open(const char* zip_path, int zip_fd, ZipArchiveHandle* handle); private: DISALLOW_COPY_AND_ASSIGN(ZipArchiveCache); std::unordered_map cache_; }; -bool ZipArchiveCache::get_or_open(const char* zip_path, ZipArchiveHandle* handle) { +bool ZipArchiveCache::get_or_open(const char* zip_path, int zip_fd, ZipArchiveHandle* handle) { std::string key(zip_path); auto it = cache_.find(key); @@ -858,7 +920,7 @@ bool ZipArchiveCache::get_or_open(const char* zip_path, ZipArchiveHandle* handle return true; } - int fd = TEMP_FAILURE_RETRY(open(zip_path, O_RDONLY | O_CLOEXEC)); + int fd = zip_fd != -1 ? dup(zip_fd) : TEMP_FAILURE_RETRY(open(zip_path, O_RDONLY | O_CLOEXEC)); if (fd == -1) { return false; } @@ -909,13 +971,19 @@ static int open_library_in_zipfile(ZipArchiveCache* zip_archive_cache, const char* zip_path = buf; const char* file_path = &buf[separator - path + 2]; - int fd = TEMP_FAILURE_RETRY(open(zip_path, O_RDONLY | O_CLOEXEC)); + int fd; + if (!strncmp("/proc/self/fd/", zip_path, strlen("/proc/self/fd/")) && + sscanf(zip_path, "/proc/self/fd/%d", &fd) == 1) { + fd = dup(fd); + } else { + fd = TEMP_FAILURE_RETRY(open(zip_path, O_RDONLY | O_CLOEXEC)); + } if (fd == -1) { return -1; } ZipArchiveHandle handle; - if (!zip_archive_cache->get_or_open(zip_path, &handle)) { + if (!zip_archive_cache->get_or_open(zip_path, fd, &handle)) { // invalid zip-file (?) close(fd); return -1; @@ -1269,6 +1337,12 @@ static bool load_library(android_namespace_t* ns, } #endif +#ifdef LD_SHIM_LIBS + for_each_matching_shim(realpath.c_str(), [&](const char* name) { + load_tasks->push_back(LoadTask::create(name, si, ns, task->get_readers_map())); + }); +#endif + for_each_dt_needed(task->get_elf_reader(), [&](const char* name) { LD_LOG(kLogDlopen, "load_library(ns=%s, task=%s): Adding DT_NEEDED task: %s", ns->get_name(), task->get_name(), name); @@ -2158,6 +2232,9 @@ void* do_dlopen(const char* name, int flags, } ProtectedDataGuard guard; +#ifdef LD_SHIM_LIBS + reset_g_active_shim_libs(); +#endif soinfo* si = find_library(ns, translated_name, flags, extinfo, caller); loading_trace.End(); diff --git a/linker/linker.h b/linker/linker.h index a80342479c..17126c6dde 100644 --- a/linker/linker.h +++ b/linker/linker.h @@ -42,6 +42,10 @@ #include "linker_logger.h" #include "linker_soinfo.h" +#ifdef LD_SHIM_LIBS +#include "linker_debug.h" +#endif + #include #include @@ -81,6 +85,22 @@ soinfo* find_containing_library(const void* p); int open_executable(const char* path, off64_t* file_offset, std::string* realpath); +#ifdef LD_SHIM_LIBS +typedef std::pair ShimDescriptor; +void parse_LD_SHIM_LIBS(const char* path); +std::vector shim_matching_pairs(const char* path); + +template +void for_each_matching_shim(const char* path, F action) { + if (path == nullptr) return; + INFO("Finding shim libs for \"%s\"", path); + for (const auto& one_pair : shim_matching_pairs(path)) { + INFO("Injecting shim lib \"%s\" as needed for %s", one_pair->second.c_str(), path); + action(one_pair->second.c_str()); + } +} +#endif + void do_android_get_LD_LIBRARY_PATH(char*, size_t); void do_android_update_LD_LIBRARY_PATH(const char* ld_library_path); void* do_dlopen(const char* name, diff --git a/linker/linker_debuggerd_android.cpp b/linker/linker_debuggerd_android.cpp index cba6345c1b..3d64628581 100644 --- a/linker/linker_debuggerd_android.cpp +++ b/linker/linker_debuggerd_android.cpp @@ -43,6 +43,7 @@ static debugger_process_info get_process_info() { .scudo_stack_depot = __libc_shared_globals()->scudo_stack_depot, .scudo_region_info = __libc_shared_globals()->scudo_region_info, .scudo_ring_buffer = __libc_shared_globals()->scudo_ring_buffer, + .scudo_ring_buffer_size = __libc_shared_globals()->scudo_ring_buffer_size, }; } #endif diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp index 9e5be345db..73947e161f 100644 --- a/linker/linker_main.cpp +++ b/linker/linker_main.cpp @@ -447,6 +447,11 @@ static ElfW(Addr) linker_main(KernelArgumentBlock& args, const char* exe_to_load parse_LD_LIBRARY_PATH(ldpath_env); parse_LD_PRELOAD(ldpreload_env); +#ifdef LD_SHIM_LIBS + // Read from TARGET_LD_SHIM_LIBS + parse_LD_SHIM_LIBS(LD_SHIM_LIBS); +#endif + std::vector namespaces = init_default_namespaces(exe_info.path.c_str()); if (!si->prelink_image()) __linker_cannot_link(g_argv[0]); @@ -472,6 +477,12 @@ static ElfW(Addr) linker_main(KernelArgumentBlock& args, const char* exe_to_load ++ld_preloads_count; } +#ifdef LD_SHIM_LIBS + for_each_matching_shim(si->get_realpath(), [&](const char* name) { + needed_library_name_list.push_back(name); + }); +#endif + for_each_dt_needed(si, [&](const char* name) { needed_library_name_list.push_back(name); }); diff --git a/linker/linker_transparent_hugepage_support.cpp b/linker/linker_transparent_hugepage_support.cpp index 65ba4cd893..0631577999 100644 --- a/linker/linker_transparent_hugepage_support.cpp +++ b/linker/linker_transparent_hugepage_support.cpp @@ -39,6 +39,6 @@ bool get_transparent_hugepages_supported() { return false; } return enabled.find("[never]") == std::string::npos; - }; + }(); return transparent_hugepages_supported; } diff --git a/tests/async_safe_test.cpp b/tests/async_safe_test.cpp index f52387e97d..dc4db07dc0 100644 --- a/tests/async_safe_test.cpp +++ b/tests/async_safe_test.cpp @@ -16,6 +16,8 @@ #include +#include + #if defined(__BIONIC__) #include #endif // __BIONIC__ @@ -227,3 +229,19 @@ TEST(async_safe_log, buffer_overrun) { GTEST_SKIP() << "bionic-only test"; #endif // __BIONIC__ } + +// Verify that using %m is never cut off. +TEST(async_safe_format_buffer, percent_m_fits_in_buffer) { +#if defined(__BIONIC__) + for (int i = 0; i < 256; i++) { + errno = i; + char async_buf[256]; + async_safe_format_buffer(async_buf, sizeof(async_buf), "%m"); + char strerror_buf[1024]; + strerror_r(errno, strerror_buf, sizeof(strerror_buf)); + ASSERT_STREQ(strerror_buf, async_buf); + } +#else // __BIONIC__ + GTEST_SKIP() << "bionic-only test"; +#endif // __BIONIC__ +} diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp index 69f8506fdf..1e58d826db 100644 --- a/tests/malloc_test.cpp +++ b/tests/malloc_test.cpp @@ -35,7 +35,11 @@ #include #include +#include +#include #include +#include +#include #include #include @@ -661,13 +665,13 @@ TEST(malloc, verify_alignment) { } TEST(malloc, mallopt_smoke) { -#if !defined(ANDROID_HOST_MUSL) +#if defined(__BIONIC__) errno = 0; ASSERT_EQ(0, mallopt(-1000, 1)); // mallopt doesn't set errno. ASSERT_EQ(0, errno); #else - GTEST_SKIP() << "musl doesn't have mallopt"; + GTEST_SKIP() << "bionic-only test"; #endif } @@ -694,6 +698,44 @@ TEST(malloc, mallopt_purge) { #endif } +TEST(malloc, mallopt_purge_all) { +#if defined(__BIONIC__) + SKIP_WITH_HWASAN << "hwasan does not implement mallopt"; + errno = 0; + ASSERT_EQ(1, mallopt(M_PURGE_ALL, 0)); +#else + GTEST_SKIP() << "bionic-only test"; +#endif +} + +// Verify that all of the mallopt values are unique. +TEST(malloc, mallopt_unique_params) { +#if defined(__BIONIC__) + std::vector> params{ + std::make_pair(M_DECAY_TIME, "M_DECAY_TIME"), + std::make_pair(M_PURGE, "M_PURGE"), + std::make_pair(M_PURGE_ALL, "M_PURGE_ALL"), + std::make_pair(M_MEMTAG_TUNING, "M_MEMTAG_TUNING"), + std::make_pair(M_THREAD_DISABLE_MEM_INIT, "M_THREAD_DISABLE_MEM_INIT"), + std::make_pair(M_CACHE_COUNT_MAX, "M_CACHE_COUNT_MAX"), + std::make_pair(M_CACHE_SIZE_MAX, "M_CACHE_SIZE_MAX"), + std::make_pair(M_TSDS_COUNT_MAX, "M_TSDS_COUNT_MAX"), + std::make_pair(M_BIONIC_ZERO_INIT, "M_BIONIC_ZERO_INIT"), + std::make_pair(M_BIONIC_SET_HEAP_TAGGING_LEVEL, "M_BIONIC_SET_HEAP_TAGGING_LEVEL"), + }; + + std::unordered_map all_params; + for (const auto& param : params) { + EXPECT_TRUE(all_params.count(param.first) == 0) + << "mallopt params " << all_params[param.first] << " and " << param.second + << " have the same value " << param.first; + all_params.insert(param); + } +#else + GTEST_SKIP() << "bionic-only test"; +#endif +} + #if defined(__BIONIC__) static void GetAllocatorVersion(bool* allocator_scudo) { TemporaryFile tf; @@ -1519,3 +1561,133 @@ TEST(malloc, realloc_mte_crash_b206701345) { } } } + +void VerifyAllocationsAreZero(std::function alloc_func, std::string function_name, + std::vector& test_sizes, size_t max_allocations) { + // Vector of zero'd data used for comparisons. Make it twice the larges size. + std::vector zero(test_sizes.back() * 2, 0); + + SCOPED_TRACE(testing::Message() << function_name << " failed to zero memory"); + + for (size_t test_size : test_sizes) { + std::vector ptrs(max_allocations); + for (size_t i = 0; i < ptrs.size(); i++) { + SCOPED_TRACE(testing::Message() << "size " << test_size << " at iteration " << i); + ptrs[i] = alloc_func(test_size); + ASSERT_TRUE(ptrs[i] != nullptr); + size_t alloc_size = malloc_usable_size(ptrs[i]); + ASSERT_LE(alloc_size, zero.size()); + ASSERT_EQ(0, memcmp(ptrs[i], zero.data(), alloc_size)); + + // Set the memory to non-zero to make sure if the pointer + // is reused it's still zero. + memset(ptrs[i], 0xab, alloc_size); + } + // Free the pointers. + for (size_t i = 0; i < ptrs.size(); i++) { + free(ptrs[i]); + } + for (size_t i = 0; i < ptrs.size(); i++) { + SCOPED_TRACE(testing::Message() << "size " << test_size << " at iteration " << i); + ptrs[i] = malloc(test_size); + ASSERT_TRUE(ptrs[i] != nullptr); + size_t alloc_size = malloc_usable_size(ptrs[i]); + ASSERT_LE(alloc_size, zero.size()); + ASSERT_EQ(0, memcmp(ptrs[i], zero.data(), alloc_size)); + } + // Free all of the pointers later to maximize the chance of reusing from + // the first loop. + for (size_t i = 0; i < ptrs.size(); i++) { + free(ptrs[i]); + } + } +} + +// Verify that small and medium allocations are always zero. +TEST(malloc, zeroed_allocations_small_medium_sizes) { + constexpr size_t kMaxAllocations = 1024; + std::vector test_sizes = {16, 48, 128, 1024, 4096, 65536}; + VerifyAllocationsAreZero([](size_t size) -> void* { return malloc(size); }, "malloc", test_sizes, + kMaxAllocations); + + VerifyAllocationsAreZero([](size_t size) -> void* { return memalign(64, size); }, "memalign", + test_sizes, kMaxAllocations); + + VerifyAllocationsAreZero( + [](size_t size) -> void* { + void* ptr; + if (posix_memalign(&ptr, 64, size) == 0) { + return ptr; + } + return nullptr; + }, + "posix_memalign", test_sizes, kMaxAllocations); +} + +// Verify that large allocations are always zero. +TEST(malloc, zeroed_allocations_large_sizes) { + constexpr size_t kMaxAllocations = 20; + std::vector test_sizes = {1000000, 2000000, 3000000, 4000000}; + VerifyAllocationsAreZero([](size_t size) -> void* { return malloc(size); }, "malloc", test_sizes, + kMaxAllocations); + + VerifyAllocationsAreZero([](size_t size) -> void* { return memalign(64, size); }, "memalign", + test_sizes, kMaxAllocations); + + VerifyAllocationsAreZero( + [](size_t size) -> void* { + void* ptr; + if (posix_memalign(&ptr, 64, size) == 0) { + return ptr; + } + return nullptr; + }, + "posix_memalign", test_sizes, kMaxAllocations); +} + +TEST(malloc, zeroed_allocations_realloc) { + // Vector of zero'd data used for comparisons. + constexpr size_t kMaxMemorySize = 131072; + std::vector zero(kMaxMemorySize, 0); + + constexpr size_t kMaxAllocations = 1024; + std::vector test_sizes = {16, 48, 128, 1024, 4096, 65536}; + // Do a number of allocations and set them to non-zero. + for (size_t test_size : test_sizes) { + std::vector ptrs(kMaxAllocations); + for (size_t i = 0; i < kMaxAllocations; i++) { + ptrs[i] = malloc(test_size); + ASSERT_TRUE(ptrs[i] != nullptr); + + // Set the memory to non-zero to make sure if the pointer + // is reused it's still zero. + memset(ptrs[i], 0xab, malloc_usable_size(ptrs[i])); + } + // Free the pointers. + for (size_t i = 0; i < kMaxAllocations; i++) { + free(ptrs[i]); + } + } + + // Do the reallocs to a larger size and verify the rest of the allocation + // is zero. + constexpr size_t kInitialSize = 8; + for (size_t test_size : test_sizes) { + std::vector ptrs(kMaxAllocations); + for (size_t i = 0; i < kMaxAllocations; i++) { + ptrs[i] = malloc(kInitialSize); + ASSERT_TRUE(ptrs[i] != nullptr); + size_t orig_alloc_size = malloc_usable_size(ptrs[i]); + + ptrs[i] = realloc(ptrs[i], test_size); + ASSERT_TRUE(ptrs[i] != nullptr); + size_t new_alloc_size = malloc_usable_size(ptrs[i]); + char* ptr = reinterpret_cast(ptrs[i]); + ASSERT_EQ(0, memcmp(&ptr[orig_alloc_size], zero.data(), new_alloc_size - orig_alloc_size)) + << "realloc from " << kInitialSize << " to size " << test_size << " at iteration " << i; + } + for (size_t i = 0; i < kMaxAllocations; i++) { + free(ptrs[i]); + } + } +}