Add unique identifier to configmetricsreportlist
1. Add incrementing report number to both configmetricsreportlist and
statsdstats
2. Add hash to both configmetricsreportlist and statsdstats
3. Print hash in print-stats
4. Move init_random_seed to main.cpp
Ignore-AOSP-First: merge conflict
Bug: 293200782
Test: atest statsd_test
Change-Id: Idb7bce7ec39c66248f44d3c1b719f0b89b44c082
Merged-In: Idb7bce7ec39c66248f44d3c1b719f0b89b44c082
(cherry picked from commit 96554d2c5ab9dc93c82284a9681e093479bef96b)
diff --git a/statsd/src/StatsLogProcessor.cpp b/statsd/src/StatsLogProcessor.cpp
index 1f697c7..e8260ad 100644
--- a/statsd/src/StatsLogProcessor.cpp
+++ b/statsd/src/StatsLogProcessor.cpp
@@ -63,6 +63,8 @@
// for ConfigKey
const int FIELD_ID_UID = 1;
const int FIELD_ID_ID = 2;
+const int FIELD_ID_REPORT_NUMBER = 3;
+const int FIELD_ID_STATSD_STATS_ID = 4;
// for ConfigMetricsReport
// const int FIELD_ID_METRICS = 1; // written in MetricsManager.cpp
const int FIELD_ID_UID_MAP = 2;
@@ -693,6 +695,18 @@
} else {
ALOGW("Config source %s does not exist", key.ToString().c_str());
}
+
+ if (erase_data) {
+ ++mDumpReportNumbers[key];
+ }
+ proto->write(FIELD_TYPE_INT32 | FIELD_ID_REPORT_NUMBER, mDumpReportNumbers[key]);
+
+ proto->write(FIELD_TYPE_INT32 | FIELD_ID_STATSD_STATS_ID,
+ StatsdStats::getInstance().getStatsdStatsId());
+ if (erase_data) {
+ StatsdStats::getInstance().noteMetricsReportSent(key, proto->size(),
+ mDumpReportNumbers[key]);
+ }
}
/*
@@ -711,8 +725,6 @@
flushProtoToBuffer(proto, outData);
VLOG("output data size %zu", outData->size());
}
-
- StatsdStats::getInstance().noteMetricsReportSent(key, proto.size());
}
/*
@@ -844,6 +856,8 @@
StatsdStats::getInstance().noteConfigRemoved(key);
mLastBroadcastTimes.erase(key);
+ mLastByteSizeTimes.erase(key);
+ mDumpReportNumbers.erase(key);
int uid = key.GetUid();
bool lastConfigForUid = true;
diff --git a/statsd/src/StatsLogProcessor.h b/statsd/src/StatsLogProcessor.h
index 4841263..77463d9 100644
--- a/statsd/src/StatsLogProcessor.h
+++ b/statsd/src/StatsLogProcessor.h
@@ -202,6 +202,9 @@
// Tracks when we last checked the bytes consumed for each config key.
std::unordered_map<ConfigKey, int64_t> mLastByteSizeTimes;
+ // Tracks the number of times a config with a specified config key has been dumped.
+ std::unordered_map<ConfigKey, int32_t> mDumpReportNumbers;
+
// Tracks when we last checked the ttl for restricted metrics.
int64_t mLastTtlTime;
diff --git a/statsd/src/StatsService.cpp b/statsd/src/StatsService.cpp
index 00152ea..ed717a3 100644
--- a/statsd/src/StatsService.cpp
+++ b/statsd/src/StatsService.cpp
@@ -230,8 +230,6 @@
init_system_properties();
- init_seed_random();
-
if (mEventQueue != nullptr) {
std::thread pushedEventThread([this] { readLogs(); });
pushedEventThread.detach();
@@ -266,18 +264,6 @@
}
}
-void StatsService::init_seed_random() {
- unsigned int seed = 0;
- // getrandom() reads bytes from urandom source into buf. If getrandom()
- // is unable to read from urandom source, then it returns -1 and we set
- // out seed to be time(nullptr) as a fallback.
- if (TEMP_FAILURE_RETRY(
- getrandom(static_cast<void*>(&seed), sizeof(unsigned int), GRND_NONBLOCK)) < 0) {
- seed = time(nullptr);
- }
- srand(seed);
-}
-
void StatsService::init_build_type_callback(void* cookie, const char* /*name*/, const char* value,
uint32_t serial) {
if (0 == strcmp("eng", value) || 0 == strcmp("userdebug", value)) {
diff --git a/statsd/src/StatsService.h b/statsd/src/StatsService.h
index a10a15e..4026417 100644
--- a/statsd/src/StatsService.h
+++ b/statsd/src/StatsService.h
@@ -266,11 +266,6 @@
void init_system_properties();
/**
- * Initiates srand for randomness.
- */
- static void init_seed_random();
-
- /**
* Helper for loading system properties.
*/
static void init_build_type_callback(void* cookie, const char* name, const char* value,
diff --git a/statsd/src/guardrail/StatsdStats.cpp b/statsd/src/guardrail/StatsdStats.cpp
index 8e3fb7f..3e38977 100644
--- a/statsd/src/guardrail/StatsdStats.cpp
+++ b/statsd/src/guardrail/StatsdStats.cpp
@@ -58,6 +58,7 @@
const int FIELD_ID_ACTIVATION_BROADCAST_GUARDRAIL = 19;
const int FIELD_ID_RESTRICTED_METRIC_QUERY_STATS = 20;
const int FIELD_ID_SHARD_OFFSET = 21;
+const int FIELD_ID_STATSD_STATS_ID = 22;
const int FIELD_ID_RESTRICTED_METRIC_QUERY_STATS_CALLING_UID = 1;
const int FIELD_ID_RESTRICTED_METRIC_QUERY_STATS_CONFIG_ID = 2;
@@ -121,6 +122,7 @@
const int FIELD_ID_CONFIG_STATS_RESTRICTED_CONFIG_FLUSH_LATENCY = 28;
const int FIELD_ID_CONFIG_STATS_RESTRICTED_CONFIG_DB_SIZE_TIME_SEC = 29;
const int FIELD_ID_CONFIG_STATS_RESTRICTED_CONFIG_DB_SIZE_BYTES = 30;
+const int FIELD_ID_CONFIG_STATS_DUMP_REPORT_NUMBER = 31;
const int FIELD_ID_INVALID_CONFIG_REASON_ENUM = 1;
const int FIELD_ID_INVALID_CONFIG_REASON_METRIC_ID = 2;
@@ -162,7 +164,7 @@
{util::CPU_TIME_PER_UID_FREQ, {6000, 10000}},
};
-StatsdStats::StatsdStats() {
+StatsdStats::StatsdStats() : mStatsdStatsId(rand()) {
mPushedAtomStats.resize(kMaxPushedAtomId + 1);
mStartTimeSec = getWallClockSec();
}
@@ -344,22 +346,24 @@
it->second->data_drop_bytes.push_back(totalBytes);
}
-void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes) {
- noteMetricsReportSent(key, num_bytes, getWallClockSec());
+void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t numBytes,
+ const int32_t reportNumber) {
+ noteMetricsReportSent(key, numBytes, getWallClockSec(), reportNumber);
}
-void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes,
- int32_t timeSec) {
+void StatsdStats::noteMetricsReportSent(const ConfigKey& key, const size_t numBytes,
+ int32_t timeSec, const int32_t reportNumber) {
lock_guard<std::mutex> lock(mLock);
auto it = mConfigStats.find(key);
if (it == mConfigStats.end()) {
ALOGE("Config key %s not found!", key.ToString().c_str());
return;
}
+
if (it->second->dump_report_stats.size() == kMaxTimestampCount) {
it->second->dump_report_stats.pop_front();
}
- it->second->dump_report_stats.push_back(std::make_pair(timeSec, num_bytes));
+ it->second->dump_report_stats.emplace_back(timeSec, numBytes, reportNumber);
}
void StatsdStats::noteDeviceInfoTableCreationFailed(const ConfigKey& key) {
@@ -1010,9 +1014,10 @@
}
for (const auto& dump : configStats->dump_report_stats) {
- dprintf(out, "\tdump report time: %s(%lld) bytes: %lld\n",
- buildTimeString(dump.first).c_str(), (long long)dump.first,
- (long long)dump.second);
+ dprintf(out, "\tdump report time: %s(%lld) bytes: %d reportNumber: %d\n",
+ buildTimeString(dump.mDumpReportTimeSec).c_str(),
+ (long long)dump.mDumpReportTimeSec, dump.mDumpReportSizeBytes,
+ dump.mDumpReportNumber);
}
for (const auto& stats : pair.second->matcher_stats) {
@@ -1173,6 +1178,8 @@
}
}
}
+ dprintf(out, "********Statsd Stats Id***********\n");
+ dprintf(out, "Statsd Stats Id %d\n", mStatsdStatsId);
dprintf(out, "********Shard Offset Provider stats***********\n");
dprintf(out, "Shard Offset: %u\n", ShardOffsetProvider::getInstance().getShardOffset());
}
@@ -1262,15 +1269,21 @@
}
for (const auto& dump : configStats.dump_report_stats) {
- proto->write(FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_DUMP_REPORT_TIME |
- FIELD_COUNT_REPEATED,
- dump.first);
+ proto->write(
+ FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_DUMP_REPORT_TIME | FIELD_COUNT_REPEATED,
+ dump.mDumpReportTimeSec);
}
for (const auto& dump : configStats.dump_report_stats) {
- proto->write(FIELD_TYPE_INT64 | FIELD_ID_CONFIG_STATS_DUMP_REPORT_BYTES |
- FIELD_COUNT_REPEATED,
- (long long)dump.second);
+ proto->write(
+ FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_DUMP_REPORT_BYTES | FIELD_COUNT_REPEATED,
+ (long long)dump.mDumpReportSizeBytes);
+ }
+
+ for (const auto& dump : configStats.dump_report_stats) {
+ proto->write(
+ FIELD_TYPE_INT32 | FIELD_ID_CONFIG_STATS_DUMP_REPORT_NUMBER | FIELD_COUNT_REPEATED,
+ dump.mDumpReportNumber);
}
for (const auto& annotation : configStats.annotations) {
@@ -1520,6 +1533,8 @@
proto.write(FIELD_TYPE_UINT32 | FIELD_ID_SHARD_OFFSET,
static_cast<long>(ShardOffsetProvider::getInstance().getShardOffset()));
+ proto.write(FIELD_TYPE_INT32 | FIELD_ID_STATSD_STATS_ID, mStatsdStatsId);
+
output->clear();
size_t bufferSize = proto.size();
output->resize(bufferSize);
diff --git a/statsd/src/guardrail/StatsdStats.h b/statsd/src/guardrail/StatsdStats.h
index 4bf6ffd..8b06f08 100644
--- a/statsd/src/guardrail/StatsdStats.h
+++ b/statsd/src/guardrail/StatsdStats.h
@@ -73,6 +73,17 @@
int64_t categoryChangedCount = 0;
} RestrictedMetricStats;
+struct DumpReportStats {
+ DumpReportStats(int32_t dumpReportSec, int32_t dumpReportSize, int32_t reportNumber)
+ : mDumpReportTimeSec(dumpReportSec),
+ mDumpReportSizeBytes(dumpReportSize),
+ mDumpReportNumber(reportNumber) {
+ }
+ int32_t mDumpReportTimeSec = 0;
+ int32_t mDumpReportSizeBytes = 0;
+ int32_t mDumpReportNumber = 0;
+};
+
struct ConfigStats {
int32_t uid;
int64_t id;
@@ -101,7 +112,8 @@
std::list<int32_t> data_drop_time_sec;
// Number of bytes dropped at corresponding time.
std::list<int64_t> data_drop_bytes;
- std::list<std::pair<int32_t, int64_t>> dump_report_stats;
+
+ std::list<DumpReportStats> dump_report_stats;
// Stores how many times a matcher have been matched. The map size is capped by kMaxConfigCount.
std::map<const int64_t, int> matcher_stats;
@@ -316,7 +328,8 @@
*
* The report may be requested via StatsManager API, or through adb cmd.
*/
- void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes);
+ void noteMetricsReportSent(const ConfigKey& key, const size_t numBytes,
+ const int32_t reportNumber);
/**
* Report failure in creating the device info metadata table for restricted configs.
@@ -637,6 +650,14 @@
*/
static std::pair<size_t, size_t> getAtomDimensionKeySizeLimits(const int atomId = -1);
+ /**
+ * Return the unique identifier for the statsd stats report. This id is
+ * reset on boot.
+ */
+ inline int32_t getStatsdStatsId() const {
+ return mStatsdStatsId;
+ }
+
typedef struct PullTimeoutMetadata {
int64_t pullTimeoutUptimeMillis;
int64_t pullTimeoutElapsedMillis;
@@ -691,6 +712,11 @@
int32_t mStartTimeSec;
+ // Random id set using rand() during the initialization. Used to uniquely
+ // identify a session. This is more reliable than mStartTimeSec due to the
+ // unreliable nature of wall clock times.
+ const int32_t mStatsdStatsId;
+
// Track the number of dropped entries used by the uid map.
UidMapStats mUidMapStats;
@@ -825,7 +851,8 @@
void noteDataDropped(const ConfigKey& key, const size_t totalBytes, int32_t timeSec);
- void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes, int32_t timeSec);
+ void noteMetricsReportSent(const ConfigKey& key, const size_t numBytes, int32_t timeSec,
+ const int32_t reportNumber);
void noteBroadcastSent(const ConfigKey& key, int32_t timeSec);
diff --git a/statsd/src/main.cpp b/statsd/src/main.cpp
index 14b31a0..6f8c6e5 100644
--- a/statsd/src/main.cpp
+++ b/statsd/src/main.cpp
@@ -23,6 +23,7 @@
#include <android/binder_manager.h>
#include <android/binder_process.h>
#include <stdio.h>
+#include <sys/random.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
@@ -66,6 +67,18 @@
sigaction(SIGTERM, &sa, nullptr);
}
+void initSeedRandom() {
+ unsigned int seed = 0;
+ // getrandom() reads bytes from urandom source into buf. If getrandom()
+ // is unable to read from urandom source, then it returns -1 and we set
+ // out seed to be time(nullptr) as a fallback.
+ if (TEMP_FAILURE_RETRY(
+ getrandom(static_cast<void*>(&seed), sizeof(unsigned int), GRND_NONBLOCK)) < 0) {
+ seed = time(nullptr);
+ }
+ srand(seed);
+}
+
int main(int /*argc*/, char** /*argv*/) {
// Set up the looper
sp<Looper> looper(Looper::prepare(0 /* opts */));
@@ -92,6 +105,7 @@
STATSD_INIT_COMPLETED_NO_DELAY_FLAG, FLAG_FALSE)
? 0
: StatsService::kStatsdInitDelaySecs;
+ initSeedRandom();
// Create the service
gStatsService =
SharedRefBase::make<StatsService>(uidMap, eventQueue, logEventFilter, initEventDelay);
diff --git a/statsd/src/stats_log.proto b/statsd/src/stats_log.proto
index 44add93..15a36c6 100644
--- a/statsd/src/stats_log.proto
+++ b/statsd/src/stats_log.proto
@@ -436,6 +436,10 @@
repeated ConfigMetricsReport reports = 2;
+ optional int32 report_number = 3;
+
+ optional int32 statsd_stats_id = 4;
+
reserved 10 to 13, 101;
}
@@ -510,6 +514,7 @@
repeated int64 restricted_flush_latency = 28;
repeated int64 restricted_db_size_time_sec = 29;
repeated int64 restricted_db_size_bytes = 30;
+ repeated int32 dump_report_number = 31;
}
repeated ConfigStats config_stats = 3;
@@ -665,6 +670,7 @@
}
repeated RestrictedMetricQueryStats restricted_metric_query_stats = 20;
optional uint32 shard_offset = 21;
+ optional int32 statsd_stats_id = 22;
}
message AlertTriggerDetails {
diff --git a/statsd/tests/e2e/EventMetric_e2e_test.cpp b/statsd/tests/e2e/EventMetric_e2e_test.cpp
index 81a6d91..a6763af 100644
--- a/statsd/tests/e2e/EventMetric_e2e_test.cpp
+++ b/statsd/tests/e2e/EventMetric_e2e_test.cpp
@@ -257,6 +257,42 @@
EXPECT_THAT(atom.repeated_enum_field(), ElementsAreArray(enumArrayMatch));
}
+TEST_F(EventMetricE2eTest, TestDumpReportIncrementsReportNumber) {
+ StatsdConfig config;
+ config.add_allowed_log_source("AID_ROOT"); // LogEvent defaults to UID of root.
+
+ AtomMatcher testAtomReportedStateFirstOnAtomMatcher =
+ CreateTestAtomRepeatedStateFirstOnAtomMatcher();
+ *config.add_atom_matcher() = testAtomReportedStateFirstOnAtomMatcher;
+
+ EventMetric testAtomReportedEventMetric = createEventMetric(
+ "EventTestAtomReported", testAtomReportedStateFirstOnAtomMatcher.id(), nullopt);
+ *config.add_event_metric() = testAtomReportedEventMetric;
+
+ ConfigKey key(123, 987);
+ uint64_t configUpdateTime = 10000000000; // 0:10
+ sp<StatsLogProcessor> processor =
+ CreateStatsLogProcessor(configUpdateTime, configUpdateTime, config, key);
+
+ uint64_t dumpTimeNs = configUpdateTime + 100 * NS_PER_SEC;
+ ConfigMetricsReportList reports;
+ vector<uint8_t> buffer;
+ processor->onDumpReport(key, dumpTimeNs, true, true, ADB_DUMP, FAST, &buffer);
+ EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ ASSERT_EQ(reports.reports_size(), 1);
+
+ EXPECT_EQ(reports.report_number(), 1);
+ EXPECT_EQ(reports.statsd_stats_id(), StatsdStats::getInstance().getStatsdStatsId());
+
+ buffer.clear();
+ processor->onDumpReport(key, dumpTimeNs + 100, true, true, ADB_DUMP, FAST, &buffer);
+ EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size()));
+ ASSERT_EQ(reports.reports_size(), 1);
+
+ EXPECT_EQ(reports.report_number(), 2);
+ EXPECT_EQ(reports.statsd_stats_id(), StatsdStats::getInstance().getStatsdStatsId());
+}
+
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif
diff --git a/statsd/tests/guardrail/StatsdStats_test.cpp b/statsd/tests/guardrail/StatsdStats_test.cpp
index a2f3280..2d831b1 100644
--- a/statsd/tests/guardrail/StatsdStats_test.cpp
+++ b/statsd/tests/guardrail/StatsdStats_test.cpp
@@ -234,9 +234,9 @@
stats.noteDataDropped(key, 123);
// dump report -> 3
- stats.noteMetricsReportSent(key, 0);
- stats.noteMetricsReportSent(key, 0);
- stats.noteMetricsReportSent(key, 0);
+ stats.noteMetricsReportSent(key, 0, 1);
+ stats.noteMetricsReportSent(key, 0, 2);
+ stats.noteMetricsReportSent(key, 0, 3);
// activation_time_sec -> 2
stats.noteActiveStatusChanged(key, true);
@@ -258,6 +258,10 @@
EXPECT_EQ(123, configReport.data_drop_bytes(0));
ASSERT_EQ(3, configReport.dump_report_time_sec_size());
ASSERT_EQ(3, configReport.dump_report_data_size_size());
+ ASSERT_EQ(3, configReport.dump_report_number_size());
+ EXPECT_EQ(1, configReport.dump_report_number(0));
+ EXPECT_EQ(2, configReport.dump_report_number(1));
+ EXPECT_EQ(3, configReport.dump_report_number(2));
ASSERT_EQ(2, configReport.activation_time_sec_size());
ASSERT_EQ(1, configReport.deactivation_time_sec_size());
ASSERT_EQ(1, configReport.annotation_size());
@@ -597,7 +601,7 @@
for (int i = 0; i < StatsdStats::kMaxTimestampCount; i++) {
stats.noteDataDropped(key, timestamps[i]);
stats.noteBroadcastSent(key, timestamps[i]);
- stats.noteMetricsReportSent(key, 0, timestamps[i]);
+ stats.noteMetricsReportSent(key, 0, timestamps[i], i + 1);
stats.noteActiveStatusChanged(key, true, timestamps[i]);
stats.noteActiveStatusChanged(key, false, timestamps[i]);
}
@@ -607,7 +611,7 @@
// now it should trigger removing oldest timestamp
stats.noteDataDropped(key, 123, 10000);
stats.noteBroadcastSent(key, 10000);
- stats.noteMetricsReportSent(key, 0, 10000);
+ stats.noteMetricsReportSent(key, 0, 10000, 21);
stats.noteActiveStatusChanged(key, true, 10000);
stats.noteActiveStatusChanged(key, false, 10000);
@@ -624,7 +628,7 @@
// the oldest timestamp is the second timestamp in history
EXPECT_EQ(1, configStats->broadcast_sent_time_sec.front());
EXPECT_EQ(1, configStats->data_drop_bytes.front());
- EXPECT_EQ(1, configStats->dump_report_stats.front().first);
+ EXPECT_EQ(1, configStats->dump_report_stats.front().mDumpReportTimeSec);
EXPECT_EQ(1, configStats->activation_time_sec.front());
EXPECT_EQ(1, configStats->deactivation_time_sec.front());
@@ -632,7 +636,7 @@
EXPECT_EQ(newTimestamp, configStats->broadcast_sent_time_sec.back());
EXPECT_EQ(newTimestamp, configStats->data_drop_time_sec.back());
EXPECT_EQ(123, configStats->data_drop_bytes.back());
- EXPECT_EQ(newTimestamp, configStats->dump_report_stats.back().first);
+ EXPECT_EQ(newTimestamp, configStats->dump_report_stats.back().mDumpReportTimeSec);
EXPECT_EQ(newTimestamp, configStats->activation_time_sec.back());
EXPECT_EQ(newTimestamp, configStats->deactivation_time_sec.back());
}
