| /* | |
| * Copyright (C) 2023 The Android Open Source Project | |
| * | |
| * Licensed under the Apache License, Version 2.0 (the "License"); | |
| * you may not use this file except in compliance with the License. | |
| * You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| #include <string.h> | |
| #include <sys/mman.h> | |
| #include <sys/syscall.h> | |
| #include <android-base/file.h> | |
| #include <android-base/stringprintf.h> | |
| #include <benchmark/benchmark.h> | |
| #include "util.h" | |
| enum BenchmarkType : uint8_t { | |
| kBenchmarkMmapOnly, | |
| kBenchmarkMunmapOnly, | |
| kBenchmarkAll, | |
| }; | |
| static size_t page_sz = getpagesize(); | |
| struct MmapParams { | |
| int prot; | |
| int flags; | |
| int64_t size; | |
| }; | |
| struct MprotectParams { | |
| int initial_prot; | |
| int mprotect_prot; | |
| int64_t size; | |
| }; | |
| template <BenchmarkType type> | |
| void MmapBenchmarkImpl(benchmark::State& state, const struct MmapParams& params, int fd, | |
| void* area = nullptr) { | |
| for (auto _ : state) { | |
| if (type == kBenchmarkMunmapOnly) state.PauseTiming(); | |
| void* addr = mmap(area, params.size, params.prot, params.flags, fd, 0); | |
| if (addr == MAP_FAILED) { | |
| state.SkipWithError(android::base::StringPrintf("mmap failed: %s", strerror(errno)).c_str()); | |
| break; | |
| } | |
| if (type == kBenchmarkMmapOnly) state.PauseTiming(); | |
| if (params.prot & PROT_WRITE) { | |
| MakeAllocationResident(addr, params.size, page_sz); | |
| } | |
| if (type == kBenchmarkMunmapOnly) state.ResumeTiming(); | |
| if (munmap(addr, params.size) != 0) { | |
| state.SkipWithError( | |
| android::base::StringPrintf("munmap failed: %s", strerror(errno)).c_str()); | |
| break; | |
| } | |
| if (type == kBenchmarkMmapOnly) state.ResumeTiming(); | |
| } | |
| } | |
| static void MmapBenchmark(benchmark::State& state, const struct MmapParams& params, int fd, | |
| void* area = nullptr) { | |
| MmapBenchmarkImpl<kBenchmarkAll>(state, params, fd, area); | |
| } | |
| static void MmapFixedBenchmark(benchmark::State& state, const struct MmapParams& params, int fd, | |
| size_t area_size, size_t offs) { | |
| if ((params.flags & MAP_FIXED) == 0) { | |
| state.SkipWithError("MmapFixedBenchmark called without MAP_FIXED set"); | |
| return; | |
| } | |
| // Create the mmap that will be used for the fixed mmaps. | |
| uint8_t* area = reinterpret_cast<uint8_t*>( | |
| mmap(nullptr, area_size, params.prot, params.flags & ~MAP_FIXED, fd, 0)); | |
| if (area == MAP_FAILED) { | |
| state.SkipWithError(android::base::StringPrintf("mmap failed: %s", strerror(errno)).c_str()); | |
| return; | |
| } | |
| MmapBenchmark(state, params, fd, area + offs); | |
| if (munmap(area, area_size) != 0) { | |
| state.SkipWithError(android::base::StringPrintf("munmap failed: %s", strerror(errno)).c_str()); | |
| return; | |
| } | |
| } | |
| static void MmapFileBenchmark(benchmark::State& state, const struct MmapParams& params, | |
| size_t area_size, size_t offs) { | |
| TemporaryFile tf; | |
| if (tf.fd < 0) { | |
| state.SkipWithError( | |
| android::base::StringPrintf("failed to create a temporary file: %s", strerror(errno)) | |
| .c_str()); | |
| return; | |
| } | |
| if (area_size > 0 && ftruncate(tf.fd, area_size)) { | |
| state.SkipWithError( | |
| android::base::StringPrintf("ftruncate failed: %s", strerror(errno)).c_str()); | |
| return; | |
| } | |
| if (params.flags & MAP_FIXED) { | |
| MmapFixedBenchmark(state, params, tf.fd, area_size, offs); | |
| } else { | |
| MmapBenchmark(state, params, tf.fd); | |
| } | |
| } | |
| // anon mmap | |
| static void BM_syscall_mmap_anon_rw(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS, | |
| .size = state.range(0), | |
| }; | |
| MmapBenchmark(state, params, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_rw, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_noreserve(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_NONE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, | |
| .size = state.range(0), | |
| }; | |
| MmapBenchmark(state, params, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_noreserve, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_none(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_NONE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS, | |
| .size = state.range(0), | |
| }; | |
| MmapBenchmark(state, params, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_none, "AT_ALL_PAGE_SIZES"); | |
| // anon fixed mmap | |
| static void BM_syscall_mmap_anon_rw_fixed(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, | |
| .size = state.range(0), | |
| }; | |
| MmapFixedBenchmark(state, params, -1, params.size, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_rw_fixed, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_none_fixed(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_NONE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, | |
| .size = state.range(0), | |
| }; | |
| MmapFixedBenchmark(state, params, -1, params.size, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_none_fixed, "AT_ALL_PAGE_SIZES"); | |
| // file mmap | |
| static void BM_syscall_mmap_file_rd_priv(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ, | |
| .flags = MAP_PRIVATE, | |
| .size = state.range(0), | |
| }; | |
| MmapFileBenchmark(state, params, params.size, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_rd_priv, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_file_rw_shared(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_SHARED, | |
| .size = state.range(0), | |
| }; | |
| MmapFileBenchmark(state, params, params.size, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_rw_shared, "AT_ALL_PAGE_SIZES"); | |
| // file fixed mmap | |
| static void BM_syscall_mmap_file_rw_priv_fixed_start(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_FIXED, | |
| .size = state.range(0), | |
| }; | |
| // allocate 3x area and map at the start | |
| MmapFileBenchmark(state, params, params.size * 3, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_rw_priv_fixed_start, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_file_rw_priv_fixed_mid(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_FIXED, | |
| .size = state.range(0), | |
| }; | |
| // allocate 3x area and map at the middle | |
| MmapFileBenchmark(state, params, params.size * 3, params.size); | |
| } | |
| // mapping at sub-page size offset is not supported, so run only for AT_MULTI_PAGE_SIZES | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_rw_priv_fixed_mid, "AT_MULTI_PAGE_SIZES"); | |
| static void BM_syscall_mmap_file_rw_priv_fixed_end(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_FIXED, | |
| .size = state.range(0), | |
| }; | |
| // allocate 3x area and map at the end | |
| MmapFileBenchmark(state, params, params.size * 3, params.size * 2); | |
| } | |
| // mapping at sub-page size offset is not supported, so run only for AT_MULTI_PAGE_SIZES | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_rw_priv_fixed_end, "AT_MULTI_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_mmap_only(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS, | |
| .size = state.range(0), | |
| }; | |
| MmapBenchmarkImpl<kBenchmarkMmapOnly>(state, params, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_mmap_only, "AT_MULTI_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_munmap_only(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS, | |
| .size = state.range(0), | |
| }; | |
| MmapBenchmarkImpl<kBenchmarkMunmapOnly>(state, params, 0); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_munmap_only, "AT_MULTI_PAGE_SIZES"); | |
| void MadviseBenchmark(benchmark::State& state, const struct MmapParams& params, int madvise_flags) { | |
| void* addr = mmap(nullptr, params.size, params.prot, params.flags, 0, 0); | |
| if (addr == MAP_FAILED) { | |
| state.SkipWithError(android::base::StringPrintf("mmap failed: %s", strerror(errno)).c_str()); | |
| return; | |
| } | |
| for (auto _ : state) { | |
| state.PauseTiming(); | |
| if (params.prot & PROT_WRITE) { | |
| MakeAllocationResident(addr, params.size, page_sz); | |
| } | |
| state.ResumeTiming(); | |
| madvise(addr, params.size, madvise_flags); | |
| } | |
| if (munmap(addr, params.size) != 0) { | |
| state.SkipWithError(android::base::StringPrintf("munmap failed: %s", strerror(errno)).c_str()); | |
| } | |
| } | |
| static void BM_syscall_mmap_anon_madvise_dontneed(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS, | |
| .size = state.range(0), | |
| }; | |
| MadviseBenchmark(state, params, MADV_DONTNEED); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_madvise_dontneed, "AT_MULTI_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_madvise_pageout(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS, | |
| .size = state.range(0), | |
| }; | |
| MadviseBenchmark(state, params, MADV_PAGEOUT); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_madvise_pageout, "AT_MULTI_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_madvise_free(benchmark::State& state) { | |
| struct MmapParams params = { | |
| .prot = PROT_READ | PROT_WRITE, | |
| .flags = MAP_PRIVATE | MAP_ANONYMOUS, | |
| .size = state.range(0), | |
| }; | |
| MadviseBenchmark(state, params, MADV_FREE); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_madvise_free, "AT_MULTI_PAGE_SIZES"); | |
| void MprotectBenchmark(benchmark::State& state, const struct MprotectParams& params, void* addr) { | |
| for (auto _ : state) { | |
| state.PauseTiming(); | |
| /* | |
| * Guarantee that physical memory pages are allocated for this region to prevent | |
| * segmentation fault when using mprotect to change permissions. | |
| */ | |
| if (params.initial_prot & PROT_WRITE) { | |
| MakeAllocationResident(addr, params.size, page_sz); | |
| } | |
| state.ResumeTiming(); | |
| if (mprotect(addr, params.size, params.mprotect_prot) != 0) { | |
| state.SkipWithError(android::base::StringPrintf("mprotect failed: %m")); | |
| break; | |
| } | |
| state.PauseTiming(); | |
| // Revert back to the original protection | |
| int res = mprotect(addr, params.size, params.initial_prot); | |
| state.ResumeTiming(); | |
| if (res != 0) { | |
| state.SkipWithError( | |
| android::base::StringPrintf("mprotect failed to revert to original prot: %m")); | |
| break; | |
| } | |
| } | |
| } | |
| static void MprotectBenchmarkWithMmapAnon(benchmark::State& state, | |
| const struct MprotectParams& params) { | |
| void* addr = mmap(nullptr, params.size, params.initial_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); | |
| if (addr == MAP_FAILED) { | |
| state.SkipWithError(android::base::StringPrintf("mmap failed: %m")); | |
| return; | |
| } | |
| MprotectBenchmark(state, params, addr); | |
| if (munmap(addr, params.size) != 0) | |
| state.SkipWithError(android::base::StringPrintf("munmap failed: %m")); | |
| } | |
| static void BM_syscall_mmap_anon_mprotect_rw_to_rd(benchmark::State& state) { | |
| struct MprotectParams params = { | |
| .initial_prot = PROT_READ | PROT_WRITE, | |
| .mprotect_prot = PROT_READ, | |
| .size = state.range(0), | |
| }; | |
| MprotectBenchmarkWithMmapAnon(state, params); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_mprotect_rw_to_rd, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_mprotect_rw_to_none(benchmark::State& state) { | |
| struct MprotectParams params = { | |
| .initial_prot = PROT_READ | PROT_WRITE, | |
| .mprotect_prot = PROT_NONE, | |
| .size = state.range(0), | |
| }; | |
| MprotectBenchmarkWithMmapAnon(state, params); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_mprotect_rw_to_none, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_anon_mprotect_rd_to_none(benchmark::State& state) { | |
| struct MprotectParams params = { | |
| .initial_prot = PROT_READ, | |
| .mprotect_prot = PROT_NONE, | |
| .size = state.range(0), | |
| }; | |
| MprotectBenchmarkWithMmapAnon(state, params); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_anon_mprotect_rd_to_none, "AT_ALL_PAGE_SIZES"); | |
| static void MprotectBenchmarkWithMmapFile(benchmark::State& state, | |
| const struct MprotectParams& params) { | |
| TemporaryFile tf; | |
| if (tf.fd < 0) { | |
| state.SkipWithError(android::base::StringPrintf("failed to create a temporary file: %m")); | |
| return; | |
| } | |
| if (params.size > 0 && ftruncate(tf.fd, params.size)) { | |
| state.SkipWithError(android::base::StringPrintf("ftruncate failed: %m")); | |
| return; | |
| } | |
| void* addr = mmap(nullptr, params.size, params.initial_prot, MAP_PRIVATE, tf.fd, 0); | |
| if (addr == MAP_FAILED) { | |
| state.SkipWithError(android::base::StringPrintf("mmap failed: %m")); | |
| return; | |
| } | |
| MprotectBenchmark(state, params, addr); | |
| if (munmap(addr, params.size) != 0) | |
| state.SkipWithError(android::base::StringPrintf("munmap failed: %m")); | |
| } | |
| static void BM_syscall_mmap_file_mprotect_rw_to_rd(benchmark::State& state) { | |
| struct MprotectParams params = { | |
| .initial_prot = PROT_READ | PROT_WRITE, | |
| .mprotect_prot = PROT_READ, | |
| .size = state.range(0), | |
| }; | |
| MprotectBenchmarkWithMmapFile(state, params); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_mprotect_rw_to_rd, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_file_mprotect_rw_to_none(benchmark::State& state) { | |
| struct MprotectParams params = { | |
| .initial_prot = PROT_READ | PROT_WRITE, | |
| .mprotect_prot = PROT_NONE, | |
| .size = state.range(0), | |
| }; | |
| MprotectBenchmarkWithMmapFile(state, params); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_mprotect_rw_to_none, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_file_mprotect_none_to_rw(benchmark::State& state) { | |
| struct MprotectParams params = { | |
| .initial_prot = PROT_NONE, | |
| .mprotect_prot = PROT_READ | PROT_WRITE, | |
| .size = state.range(0), | |
| }; | |
| MprotectBenchmarkWithMmapFile(state, params); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_mprotect_none_to_rw, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_file_mprotect_none_to_rd(benchmark::State& state) { | |
| struct MprotectParams params = { | |
| .initial_prot = PROT_NONE, | |
| .mprotect_prot = PROT_READ, | |
| .size = state.range(0), | |
| }; | |
| MprotectBenchmarkWithMmapFile(state, params); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_mprotect_none_to_rd, "AT_ALL_PAGE_SIZES"); | |
| static void BM_syscall_mmap_file_mprotect_rd_to_none(benchmark::State& state) { | |
| struct MprotectParams params = { | |
| .initial_prot = PROT_READ, | |
| .mprotect_prot = PROT_NONE, | |
| .size = state.range(0), | |
| }; | |
| MprotectBenchmarkWithMmapFile(state, params); | |
| } | |
| BIONIC_BENCHMARK_WITH_ARG(BM_syscall_mmap_file_mprotect_rd_to_none, "AT_ALL_PAGE_SIZES"); |