| #include "huge_page_deducer.h" |
| |
| #include <limits> |
| |
| #include "perf_data_utils.h" |
| |
| #include "base/logging.h" |
| |
| using PerfEvent = quipper::PerfDataProto::PerfEvent; |
| using MMapEvent = quipper::PerfDataProto::MMapEvent; |
| |
| namespace quipper { |
| namespace { |
| |
| const char kAnonFilename[] = "//anon"; |
| const size_t kHugepageSize = 1 << 21; |
| |
| bool IsAnon(const MMapEvent& event) { |
| return event.filename() == kAnonFilename; |
| } |
| |
| // IsContiguous returns true if mmap |a| is immediately followed by |b| |
| // within a process' address space. |
| bool IsContiguous(const MMapEvent& a, const MMapEvent& b) { |
| return a.pid() == b.pid() && (a.start() + a.len()) == b.start(); |
| } |
| |
| // IsEquivalentFile returns true iff |a| and |b| have the same name, or if |
| // either of them are anonymous memory (and thus likely to be a --hugepage_text |
| // version of the same file). |
| bool IsEquivalentFile(const MMapEvent& a, const MMapEvent& b) { |
| // perf attributes neighboring anonymous mappings under the argv[0] |
| // filename rather than "//anon", so check filename equality, as well as |
| // anonymous. |
| return a.filename() == b.filename() || IsAnon(a) || IsAnon(b); |
| } |
| |
| // Helper to correctly update a filename on a PerfEvent that contains an |
| // MMapEvent. |
| void SetMmapFilename(PerfEvent* event, const string& new_filename, |
| uint64_t new_filename_md5_prefix) { |
| CHECK(event->has_mmap_event()); |
| event->mutable_header()->set_size( |
| event->header().size() + GetUint64AlignedStringLength(new_filename) - |
| GetUint64AlignedStringLength(event->mmap_event().filename())); |
| |
| event->mutable_mmap_event()->set_filename(new_filename); |
| event->mutable_mmap_event()->set_filename_md5_prefix(new_filename_md5_prefix); |
| } |
| } // namespace |
| |
| namespace { |
| |
| // MMapRange represents an index into a PerfEvents sequence that contains |
| // a contiguous region of mmaps that have all of the same filename and pgoff. |
| class MMapRange { |
| public: |
| // Default constructor is an invalid range. |
| MMapRange() |
| : first_(std::numeric_limits<int>::max()), |
| last_(std::numeric_limits<int>::min()) {} |
| |
| // Construct a real range. |
| MMapRange(int first_index, int last_index) |
| : first_(first_index), last_(last_index) {} |
| |
| uint64 Len(const RepeatedPtrField<PerfEvent>& events) const { |
| auto& first = events.Get(first_).mmap_event(); |
| auto& last = events.Get(last_).mmap_event(); |
| return last.start() - first.start() + last.len(); |
| } |
| |
| int FirstIndex() const { return first_; } |
| int LastIndex() const { return last_; } |
| bool IsValid() const { return first_ <= last_; } |
| |
| const MMapEvent& FirstMmap(const RepeatedPtrField<PerfEvent>& events) const { |
| return events.Get(first_).mmap_event(); |
| } |
| |
| const MMapEvent& LastMmap(const RepeatedPtrField<PerfEvent>& events) const { |
| return events.Get(last_).mmap_event(); |
| } |
| |
| private: |
| int first_; |
| int last_; |
| }; |
| |
| std::ostream& operator<<(std::ostream& os, const MMapRange& r) { |
| os << "[" << r.FirstIndex() << "," << r.LastIndex() << "]"; |
| return os; |
| } |
| |
| // MMapRange version of IsContiguous(MMapEvent, MMapEvent). |
| bool IsContiguous(const RepeatedPtrField<PerfEvent>& events, const MMapRange& a, |
| const MMapRange& b) { |
| return IsContiguous(a.LastMmap(events), b.FirstMmap(events)); |
| } |
| |
| // MMapRange version of IsIsEquivalent(MMapEvent, MMapEvent). |
| bool IsEquivalentFile(const RepeatedPtrField<PerfEvent>& events, |
| const MMapRange& a, const MMapRange& b) { |
| // Because a range has the same file for all mmaps within it, assume that |
| // checking any mmap in |a| with any in |b| is sufficient. |
| return IsEquivalentFile(a.LastMmap(events), b.FirstMmap(events)); |
| } |
| |
| // FindRange returns a MMapRange of contiguous MmapEvents that: |
| // - either: |
| // - contains 1 or more MmapEvents with pgoff == 0 |
| // - is a single MmapEvent with pgoff != 0 |
| // - and: |
| // - has the same filename for all entries |
| // Otherwise, if none can be found, an invalid range will be returned. |
| MMapRange FindRange(const RepeatedPtrField<PerfEvent>& events, int start) { |
| const MMapEvent* prev_mmap = nullptr; |
| MMapRange range; |
| for (int i = start; i < events.size(); i++) { |
| const PerfEvent& event = events.Get(i); |
| // Skip irrelevant events |
| if (!event.has_mmap_event()) { |
| continue; |
| } |
| // Skip dynamic mmap() events. Hugepage deduction only works on mmaps as |
| // synthesized by perf from /proc/${pid}/maps, which have timestamp==0. |
| // Support for deducing hugepages from a sequence of mmap()/mremap() calls |
| // would require additional deduction logic. |
| if (event.timestamp() != 0) { |
| continue; |
| } |
| const MMapEvent& mmap = events.Get(i).mmap_event(); |
| if (prev_mmap == nullptr) { |
| range = MMapRange(i, i); |
| prev_mmap = &mmap; |
| } |
| // Ranges match exactly: //anon,//anon, or file,file; If they use different |
| // names, then deduction needs to consider them independently. |
| if (prev_mmap->filename() != mmap.filename()) { |
| break; |
| } |
| // If they're not virtually contiguous, they're not a single range. |
| if (start != i && !IsContiguous(*prev_mmap, mmap)) { |
| break; |
| } |
| // If this segment has a page offset, assume that it is *not* hugepage |
| // backed, and thus does not need separate deduction. |
| if (mmap.pgoff() != 0) { |
| break; |
| } |
| CHECK(mmap.pgoff() == 0 || !IsAnon(mmap)) |
| << "Anonymous pages can't have pgoff set"; |
| prev_mmap = &mmap; |
| range = MMapRange(range.FirstIndex(), i); |
| } |
| // Range has: |
| // - single file |
| // - virtually contiguous |
| // - either: is multiple mappings *or* has pgoff=0 |
| return range; |
| } |
| |
| // FindNextRange will return the next range after the given |prev_range| if |
| // there is one; otherwise it will return an invalid range. |
| MMapRange FindNextRange(const RepeatedPtrField<PerfEvent>& events, |
| const MMapRange& prev_range) { |
| MMapRange ret; |
| if (prev_range.IsValid() && prev_range.LastIndex() < events.size()) { |
| ret = FindRange(events, prev_range.LastIndex() + 1); |
| } |
| return ret; |
| } |
| |
| // UpdateRangeFromNext will set the filename / pgoff of all mmaps within |range| |
| // to be pgoff-contiguous with |next_range|, and match its file information. |
| void UpdateRangeFromNext(const MMapRange& range, const MMapRange& next_range, |
| RepeatedPtrField<PerfEvent>* events) { |
| CHECK(range.LastIndex() < events->size()); |
| CHECK(next_range.LastIndex() < events->size()); |
| const MMapEvent& src = next_range.FirstMmap(*events); |
| const uint64 start_pgoff = src.pgoff() - range.Len(*events); |
| uint64 pgoff = start_pgoff; |
| for (int i = range.FirstIndex(); i <= range.LastIndex(); i++) { |
| if (!events->Get(i).has_mmap_event()) { |
| continue; |
| } |
| PerfEvent* event = events->Mutable(i); |
| MMapEvent* mmap = event->mutable_mmap_event(); |
| |
| // Replace "//anon" with a regular name if possible. |
| if (IsAnon(*mmap)) { |
| // ANDROID-CHANGED: protobuf-lite. |
| CHECK_EQ(mmap->pgoff(), 0u) << "//anon should have offset=0 for mmap"; |
| // << event->ShortDebugString(); |
| SetMmapFilename(event, src.filename(), src.filename_md5_prefix()); |
| } |
| |
| if (mmap->pgoff() == 0) { |
| mmap->set_pgoff(pgoff); |
| if (src.has_maj()) { |
| mmap->set_maj(src.maj()); |
| } |
| if (src.has_min()) { |
| mmap->set_min(src.min()); |
| } |
| if (src.has_ino()) { |
| mmap->set_ino(src.ino()); |
| } |
| if (src.has_ino_generation()) { |
| mmap->set_ino_generation(src.ino_generation()); |
| } |
| } |
| pgoff += mmap->len(); |
| } |
| CHECK_EQ(pgoff, start_pgoff + range.Len(*events)); |
| } |
| } // namespace |
| |
| void DeduceHugePages(RepeatedPtrField<PerfEvent>* events) { |
| // |prev_range|, if IsValid(), represents the preview mmap range seen (and |
| // already processed / updated). |
| MMapRange prev_range; |
| // |range| contains the currently-being-processed mmap range, which will have |
| // its hugepages ranges deduced. |
| MMapRange range = FindRange(*events, 0); |
| // |next_range| contains the next range to process, possibily containing |
| // pgoff != 0 or !IsAnon(filename) from which the current range can be |
| // updated. |
| MMapRange next_range = FindNextRange(*events, range); |
| |
| for (; range.IsValid(); prev_range = range, range = next_range, |
| next_range = FindNextRange(*events, range)) { |
| const bool have_next = |
| (next_range.IsValid() && IsContiguous(*events, range, next_range) && |
| IsEquivalentFile(*events, range, next_range)); |
| |
| // If there's no mmap after this, then we assume that this is *not* viable |
| // a hugepage_text mapping. This is true unless we're really unlucky. If: |
| // - the binary is mapped such that the limit is hugepage aligned |
| // (presumably 4Ki/2Mi chance == p=0.03125) |
| // - and the entire binaryis hugepage_text mapped |
| if (!have_next) { |
| continue; |
| } |
| |
| const bool have_prev = |
| (prev_range.IsValid() && IsContiguous(*events, prev_range, range) && |
| IsEquivalentFile(*events, prev_range, range) && |
| IsEquivalentFile(*events, prev_range, next_range)); |
| |
| uint64 start_pgoff = 0; |
| if (have_prev) { |
| const auto& prev = prev_range.LastMmap(*events); |
| start_pgoff = prev.pgoff() + prev.len(); |
| } |
| const auto& next = next_range.FirstMmap(*events); |
| // prev.pgoff should be valid now, so let's double-check that |
| // if next has a non-zero pgoff, that {prev,curr,next} will have |
| // contiguous pgoff once updated. |
| if (next.pgoff() >= range.Len(*events) && |
| (next.pgoff() - range.Len(*events)) == start_pgoff) { |
| UpdateRangeFromNext(range, next_range, events); |
| } |
| } |
| } |
| |
| void CombineMappings(RepeatedPtrField<PerfEvent>* events) { |
| // Combine mappings |
| RepeatedPtrField<PerfEvent> new_events; |
| new_events.Reserve(events->size()); |
| |
| // |prev| is the index of the last mmap_event in |new_events| (or |
| // |new_events.size()| if no mmap_events have been inserted yet). |
| int prev = 0; |
| for (int i = 0; i < events->size(); ++i) { |
| PerfEvent* event = events->Mutable(i); |
| if (!event->has_mmap_event()) { |
| new_events.Add()->Swap(event); |
| continue; |
| } |
| |
| const MMapEvent& mmap = event->mmap_event(); |
| // Try to merge mmap with |new_events[prev]|. |
| while (prev < new_events.size() && !new_events.Get(prev).has_mmap_event()) { |
| prev++; |
| } |
| |
| if (prev >= new_events.size()) { |
| new_events.Add()->Swap(event); |
| continue; |
| } |
| |
| MMapEvent* prev_mmap = new_events.Mutable(prev)->mutable_mmap_event(); |
| |
| // Don't use IsEquivalentFile(); we don't want to combine //anon with |
| // files if DeduceHugepages didn't already fix up the mappings. |
| const bool file_match = prev_mmap->filename() == mmap.filename(); |
| const bool pgoff_contiguous = |
| file_match && (prev_mmap->pgoff() + prev_mmap->len() == mmap.pgoff()); |
| |
| const bool combine_mappings = |
| IsContiguous(*prev_mmap, mmap) && pgoff_contiguous; |
| if (!combine_mappings) { |
| new_events.Add()->Swap(event); |
| prev++; |
| continue; |
| } |
| |
| // Combine the lengths of the two mappings. |
| prev_mmap->set_len(prev_mmap->len() + mmap.len()); |
| } |
| |
| events->Swap(&new_events); |
| } |
| |
| } // namespace quipper |