Merge "UseV2 soft memory calculations" into main
diff --git a/statsd/src/guardrail/StatsdStats.cpp b/statsd/src/guardrail/StatsdStats.cpp
index 2d96ce3..16354b7 100644
--- a/statsd/src/guardrail/StatsdStats.cpp
+++ b/statsd/src/guardrail/StatsdStats.cpp
@@ -63,6 +63,7 @@
const int FIELD_ID_SUBSCRIPTION_STATS = 23;
const int FIELD_ID_SOCKET_LOSS_STATS = 24;
const int FIELD_ID_QUEUE_STATS = 25;
+const int FIELD_ID_SOCKET_READ_STATS = 26;
const int FIELD_ID_RESTRICTED_METRIC_QUERY_STATS_CALLING_UID = 1;
const int FIELD_ID_RESTRICTED_METRIC_QUERY_STATS_CONFIG_ID = 2;
@@ -201,13 +202,17 @@
const int FIELD_ID_PER_SUBSCRIPTION_STATS_END_TIME = 5;
const int FIELD_ID_PER_SUBSCRIPTION_STATS_FLUSH_COUNT = 6;
+// Socket read stats
+const int FIELD_ID_SOCKET_READ_STATS_BATCHED_READ_SIZE = 1;
+
const std::map<int, std::pair<size_t, size_t>> StatsdStats::kAtomDimensionKeySizeLimitMap = {
{util::BINDER_CALLS, {6000, 10000}},
{util::LOOPER_STATS, {1500, 2500}},
{util::CPU_TIME_PER_UID_FREQ, {6000, 10000}},
};
-StatsdStats::StatsdStats() : mStatsdStatsId(rand()) {
+StatsdStats::StatsdStats()
+ : mStatsdStatsId(rand()), mSocketBatchReadHistogram(kNumBinsInSocketBatchReadHistogram) {
mPushedAtomStats.resize(kMaxPushedAtomId + 1);
mStartTimeSec = getWallClockSec();
}
@@ -293,6 +298,28 @@
mLogLossStats.emplace_back(wallClockTimeSec, count, lastError, lastTag, uid, pid);
}
+void StatsdStats::noteBatchSocketRead(int32_t size) {
+ // Calculate the bin.
+ int bin = 0;
+ if (size < 0) {
+ ALOGE("Unexpected negative size read from socket. This should never happen");
+ bin = 0;
+ } else if (size < 5) {
+ bin = size; // bin = [0,4].
+ } else if (size < 10) {
+ bin = 4 + (size / 5); // bin = 5.
+ } else if (size < 100) {
+ bin = 5 + (size / 10); // bin = [6,14].
+ } else if (size < 1000) {
+ bin = 14 + (size / 100); // bin = [15-23].
+ } else if (size < 2000) {
+ bin = 19 + (size / 200); // bin = [24-28].
+ } else { // 2000+
+ bin = 29;
+ }
+ lock_guard<std::mutex> lock(mLock);
+ mSocketBatchReadHistogram[bin] += 1;
+}
void StatsdStats::noteBroadcastSent(const ConfigKey& key) {
noteBroadcastSent(key, getWallClockSec());
}
@@ -1112,6 +1139,7 @@
mPushedAtomDropsStats.clear();
mRestrictedMetricQueryStats.clear();
mSubscriptionPullThreadWakeupCount = 0;
+ std::fill(mSocketBatchReadHistogram.begin(), mSocketBatchReadHistogram.end(), 0);
for (auto it = mSubscriptionStats.begin(); it != mSubscriptionStats.end();) {
if (it->second.end_time_sec > 0) {
@@ -1429,6 +1457,28 @@
(long long)restart);
}
+ dprintf(out, "********Socket batch read size stats***********\n");
+ for (int i = 0; i < kNumBinsInSocketBatchReadHistogram; i++) {
+ if (mSocketBatchReadHistogram[i] == 0) {
+ continue;
+ }
+ string range;
+ if (i < 5) {
+ range = "[" + to_string(i) + "]";
+ } else if (i == 5) {
+ range = "[5-9]";
+ } else if (i < 15) {
+ range = "[" + to_string(i - 5) + "0-" + to_string(i - 5) + "9]";
+ } else if (i < 24) {
+ range = "[" + to_string(i - 14) + "00-" + to_string(i - 14) + "99]";
+ } else if (i < 29) {
+ range = "[" + to_string((i - 19) * 2) + "00-" + to_string((i - 19) * 2 + 1) + "99]";
+ } else {
+ range = "[2000+]";
+ }
+ dprintf(out, "%s: %lld\n", range.c_str(), (long long)mSocketBatchReadHistogram[i]);
+ }
+
for (const auto& loss : mLogLossStats) {
dprintf(out,
"Log loss: %lld (wall clock sec) - %d (count), %d (last error), %d (last tag), %d "
@@ -1508,6 +1558,7 @@
dprintf(out, "Statsd Stats Id %d\n", mStatsdStatsId);
dprintf(out, "********Shard Offset Provider stats***********\n");
dprintf(out, "Shard Offset: %u\n", ShardOffsetProvider::getInstance().getShardOffset());
+ dprintf(out, "\n");
}
void addConfigStatsToProto(const ConfigStats& configStats, ProtoOutputStream* proto) {
@@ -1949,6 +2000,16 @@
proto.end(socketLossStatsToken);
+ // Socket batch read stats.
+ const uint64_t socketReadStatsToken =
+ proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_SOCKET_READ_STATS);
+ for (const auto& it : mSocketBatchReadHistogram) {
+ proto.write(FIELD_TYPE_INT64 | FIELD_ID_SOCKET_READ_STATS_BATCHED_READ_SIZE |
+ FIELD_COUNT_REPEATED,
+ it);
+ }
+ proto.end(socketReadStatsToken);
+
output->clear();
proto.serializeToVector(output);
diff --git a/statsd/src/guardrail/StatsdStats.h b/statsd/src/guardrail/StatsdStats.h
index 106eed8..03266ee 100644
--- a/statsd/src/guardrail/StatsdStats.h
+++ b/statsd/src/guardrail/StatsdStats.h
@@ -307,6 +307,8 @@
static const int32_t kMaxLoggedBucketDropEvents = 10;
+ static const int32_t kNumBinsInSocketBatchReadHistogram = 30;
+
/**
* Report a new config has been received and report the static stats about the config.
*
@@ -717,6 +719,8 @@
*/
void noteSubscriptionPullThreadWakeup();
+ void noteBatchSocketRead(int32_t readSize);
+
/**
* Reset the historical stats. Including all stats in icebox, and the tracked stats about
* metrics, matchers, and atoms. The active configs will be kept and StatsdStats will continue
@@ -947,6 +951,8 @@
std::list<int32_t> mSystemServerRestartSec;
+ std::vector<int64_t> mSocketBatchReadHistogram;
+
struct RestrictedMetricQueryStats {
RestrictedMetricQueryStats(int32_t callingUid, int64_t configId,
const string& configPackage, std::optional<int32_t> configUid,
@@ -1066,6 +1072,7 @@
FRIEND_TEST(StatsdStatsTest, TestSystemServerCrash);
FRIEND_TEST(StatsdStatsTest, TestTimestampThreshold);
FRIEND_TEST(StatsdStatsTest, TestValidConfigAdd);
+ FRIEND_TEST(StatsdStatsTest, TestSocketBatchReadStats);
};
InvalidConfigReason createInvalidConfigReasonWithMatcher(const InvalidConfigReasonEnum reason,
diff --git a/statsd/src/socket/StatsSocketListener.cpp b/statsd/src/socket/StatsSocketListener.cpp
index 7f0eb43..c9f8b89 100644
--- a/statsd/src/socket/StatsSocketListener.cpp
+++ b/statsd/src/socket/StatsSocketListener.cpp
@@ -64,41 +64,45 @@
};
const int socket = cli->getSocket();
-
- // To clear the entire buffer is secure/safe, but this contributes to 1.68%
- // overhead under logging load. We are safe because we check counts, but
- // still need to clear null terminator
- // memset(buffer, 0, sizeof(buffer));
- ssize_t n = recvmsg(socket, &hdr, 0);
- if (n <= (ssize_t)(sizeof(android_log_header_t))) {
- return false;
- }
-
- buffer[n] = 0;
-
- struct ucred* cred = NULL;
-
- struct cmsghdr* cmsg = CMSG_FIRSTHDR(&hdr);
- while (cmsg != NULL) {
- if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_CREDENTIALS) {
- cred = (struct ucred*)CMSG_DATA(cmsg);
- break;
+ int i = 0;
+ ssize_t n = 0;
+ while (n = recvmsg(socket, &hdr, MSG_DONTWAIT), n > 0) {
+ // To clear the entire buffer is secure/safe, but this contributes to 1.68%
+ // overhead under logging load. We are safe because we check counts, but
+ // still need to clear null terminator.
+ // Note that the memset, if needed, should happen before each read in the while loop.
+ // memset(buffer, 0, sizeof(buffer));
+ if (n <= (ssize_t)(sizeof(android_log_header_t))) {
+ return false;
}
- cmsg = CMSG_NXTHDR(&hdr, cmsg);
+ buffer[n] = 0;
+ i++;
+
+ struct ucred* cred = NULL;
+
+ struct cmsghdr* cmsg = CMSG_FIRSTHDR(&hdr);
+ while (cmsg != NULL) {
+ if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_CREDENTIALS) {
+ cred = (struct ucred*)CMSG_DATA(cmsg);
+ break;
+ }
+ cmsg = CMSG_NXTHDR(&hdr, cmsg);
+ }
+
+ struct ucred fake_cred;
+ if (cred == NULL) {
+ cred = &fake_cred;
+ cred->pid = 0;
+ cred->uid = DEFAULT_OVERFLOWUID;
+ }
+
+ const uint32_t uid = cred->uid;
+ const uint32_t pid = cred->pid;
+
+ processSocketMessage(buffer, n, uid, pid, *mQueue, *mLogEventFilter);
}
- struct ucred fake_cred;
- if (cred == NULL) {
- cred = &fake_cred;
- cred->pid = 0;
- cred->uid = DEFAULT_OVERFLOWUID;
- }
-
- const uint32_t uid = cred->uid;
- const uint32_t pid = cred->pid;
-
- processSocketMessage(buffer, n, uid, pid, *mQueue, *mLogEventFilter);
-
+ StatsdStats::getInstance().noteBatchSocketRead(i);
return true;
}
diff --git a/statsd/src/stats_log.proto b/statsd/src/stats_log.proto
index 9d5a376..93b0d8c 100644
--- a/statsd/src/stats_log.proto
+++ b/statsd/src/stats_log.proto
@@ -698,6 +698,17 @@
}
optional EventQueueStats event_queue_stats = 25;
+
+ // Tracks info about reads from the socket.
+ message SocketReadStats {
+ // This is a histogram of sizes of the batched reads. The bins are as follows:
+ // [0, 1, 2, 3, 4, 5-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, 70-79, 80-89, 90-99,
+ // 100-199, 200-299, 300-399, 400-499, 500-599, 600-699, 700-799, 800-899, 900-999,
+ // 1000-1199, 1200-1399, 1400-1599, 1600-1799, 1800-1999, 2000+]
+ repeated int64 batched_read_size = 1;
+ }
+
+ optional SocketReadStats socket_read_stats = 26;
}
message AlertTriggerDetails {
diff --git a/statsd/tests/guardrail/StatsdStats_test.cpp b/statsd/tests/guardrail/StatsdStats_test.cpp
index a2da3c4..9232f5b 100644
--- a/statsd/tests/guardrail/StatsdStats_test.cpp
+++ b/statsd/tests/guardrail/StatsdStats_test.cpp
@@ -1114,6 +1114,47 @@
}
}
+TEST(StatsdStatsTest, TestSocketBatchReadStats) {
+ StatsdStats stats;
+ stats.noteBatchSocketRead(1); // bin 1
+ stats.noteBatchSocketRead(2); // bin 2
+ stats.noteBatchSocketRead(2); // bin 2
+ stats.noteBatchSocketRead(4); // bin 4
+ stats.noteBatchSocketRead(5); // bin 5
+ stats.noteBatchSocketRead(9); // bin 5
+ stats.noteBatchSocketRead(9); // bin 5
+ stats.noteBatchSocketRead(10); // bin 6
+ stats.noteBatchSocketRead(19); // bin 6
+ stats.noteBatchSocketRead(30); // bin 8
+ stats.noteBatchSocketRead(32); // bin 8
+ stats.noteBatchSocketRead(39); // bin 8
+ stats.noteBatchSocketRead(90); // bin 14
+ stats.noteBatchSocketRead(99); // bin 14
+ stats.noteBatchSocketRead(100); // bin 15
+ stats.noteBatchSocketRead(100); // bin 15
+ stats.noteBatchSocketRead(199); // bin 15
+ stats.noteBatchSocketRead(200); // bin 16
+ stats.noteBatchSocketRead(299); // bin 16
+ stats.noteBatchSocketRead(999); // bin 23
+ stats.noteBatchSocketRead(1000); // bin 24
+ stats.noteBatchSocketRead(1199); // bin 24
+ stats.noteBatchSocketRead(1200); // bin 25
+ stats.noteBatchSocketRead(1800); // bin 28
+ stats.noteBatchSocketRead(1999); // bin 28
+ stats.noteBatchSocketRead(2000); // bin 29
+ stats.noteBatchSocketRead(1200000); // bin 29
+
+ StatsdStatsReport report = getStatsdStatsReport(stats, /* reset stats */ false);
+ EXPECT_THAT(report.socket_read_stats().batched_read_size(),
+ ElementsAre(0, 1, 2, 0, 1, 3, 2, 0, 3, 0, 0, 0, 0, 0, 2, 3, 2, 0, 0, 0, 0, 0, 0, 1,
+ 2, 1, 0, 0, 2, 2));
+
+ stats.reset();
+ report = getStatsdStatsReport(stats, /* reset stats */ false);
+ EXPECT_THAT(report.socket_read_stats().batched_read_size(),
+ AllOf(SizeIs(StatsdStats::kNumBinsInSocketBatchReadHistogram), Each(0)));
+}
+
TEST_P(StatsdStatsTest_GetAtomDimensionKeySizeLimit_InMap, TestGetAtomDimensionKeySizeLimits) {
const auto& [atomId, defaultHardLimit] = GetParam();
EXPECT_EQ(StatsdStats::getAtomDimensionKeySizeLimits(atomId, defaultHardLimit),
