Merge "Parse gzip'ed traces"
diff --git a/Android.bp b/Android.bp
index 7be3725..a4b5fe5 100644
--- a/Android.bp
+++ b/Android.bp
@@ -3731,6 +3731,7 @@
"src/trace_processor/counter_values_table.cc",
"src/trace_processor/event_tracker.cc",
"src/trace_processor/filtered_row_index.cc",
+ "src/trace_processor/forwarding_trace_parser.cc",
"src/trace_processor/ftrace_descriptors.cc",
"src/trace_processor/ftrace_utils.cc",
"src/trace_processor/fuchsia_provider_view.cc",
@@ -3902,6 +3903,7 @@
"src/trace_processor/counter_values_table.cc",
"src/trace_processor/event_tracker.cc",
"src/trace_processor/filtered_row_index.cc",
+ "src/trace_processor/forwarding_trace_parser.cc",
"src/trace_processor/ftrace_descriptors.cc",
"src/trace_processor/ftrace_utils.cc",
"src/trace_processor/fuchsia_provider_view.cc",
diff --git a/BUILD b/BUILD
index 144c09f..6703ee1 100644
--- a/BUILD
+++ b/BUILD
@@ -299,6 +299,8 @@
"src/trace_processor/export_json.h",
"src/trace_processor/filtered_row_index.cc",
"src/trace_processor/filtered_row_index.h",
+ "src/trace_processor/forwarding_trace_parser.cc",
+ "src/trace_processor/forwarding_trace_parser.h",
"src/trace_processor/ftrace_descriptors.cc",
"src/trace_processor/ftrace_descriptors.h",
"src/trace_processor/ftrace_utils.cc",
@@ -591,6 +593,8 @@
"src/trace_processor/export_json.h",
"src/trace_processor/filtered_row_index.cc",
"src/trace_processor/filtered_row_index.h",
+ "src/trace_processor/forwarding_trace_parser.cc",
+ "src/trace_processor/forwarding_trace_parser.h",
"src/trace_processor/ftrace_descriptors.cc",
"src/trace_processor/ftrace_descriptors.h",
"src/trace_processor/ftrace_utils.cc",
@@ -835,6 +839,8 @@
"src/trace_processor/export_json.h",
"src/trace_processor/filtered_row_index.cc",
"src/trace_processor/filtered_row_index.h",
+ "src/trace_processor/forwarding_trace_parser.cc",
+ "src/trace_processor/forwarding_trace_parser.h",
"src/trace_processor/ftrace_descriptors.cc",
"src/trace_processor/ftrace_descriptors.h",
"src/trace_processor/ftrace_utils.cc",
diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn
index ef0d92f..e7b4a97 100644
--- a/src/trace_processor/BUILD.gn
+++ b/src/trace_processor/BUILD.gn
@@ -63,6 +63,8 @@
"event_tracker.h",
"filtered_row_index.cc",
"filtered_row_index.h",
+ "forwarding_trace_parser.cc",
+ "forwarding_trace_parser.h",
"ftrace_descriptors.cc",
"ftrace_descriptors.h",
"ftrace_utils.cc",
@@ -237,6 +239,7 @@
"clock_tracker_unittest.cc",
"event_tracker_unittest.cc",
"filtered_row_index_unittest.cc",
+ "forwarding_trace_parser_unittest.cc",
"ftrace_utils_unittest.cc",
"fuchsia_trace_utils_unittest.cc",
"heap_profile_tracker_unittest.cc",
@@ -254,7 +257,6 @@
"syscall_tracker_unittest.cc",
"systrace_parser_unittest.cc",
"thread_table_unittest.cc",
- "trace_processor_impl_unittest.cc",
"trace_sorter_unittest.cc",
]
deps = [
diff --git a/src/trace_processor/forwarding_trace_parser.cc b/src/trace_processor/forwarding_trace_parser.cc
new file mode 100644
index 0000000..ca657f8
--- /dev/null
+++ b/src/trace_processor/forwarding_trace_parser.cc
@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/forwarding_trace_parser.h"
+
+#include "perfetto/base/logging.h"
+#include "perfetto/ext/base/string_utils.h"
+#include "src/trace_processor/fuchsia_trace_parser.h"
+#include "src/trace_processor/fuchsia_trace_tokenizer.h"
+#include "src/trace_processor/gzip_trace_parser.h"
+#include "src/trace_processor/proto_trace_parser.h"
+#include "src/trace_processor/proto_trace_tokenizer.h"
+#include "src/trace_processor/systrace_trace_parser.h"
+
+// JSON parsing and exporting is only supported in the standalone and
+// Chromium builds.
+#if PERFETTO_BUILDFLAG(PERFETTO_STANDALONE_BUILD) || \
+ PERFETTO_BUILD_WITH_CHROMIUM
+#include "src/trace_processor/json_trace_parser.h"
+#include "src/trace_processor/json_trace_tokenizer.h"
+#endif
+
+namespace perfetto {
+namespace trace_processor {
+namespace {
+
+std::string RemoveWhitespace(const std::string& input) {
+ std::string str(input);
+ str.erase(std::remove_if(str.begin(), str.end(), ::isspace), str.end());
+ return str;
+}
+
+// Fuchsia traces have a magic number as documented here:
+// https://fuchsia.googlesource.com/fuchsia/+/HEAD/docs/development/tracing/trace-format/README.md#magic-number-record-trace-info-type-0
+constexpr uint64_t kFuchsiaMagicNumber = 0x0016547846040010;
+
+} // namespace
+
+ForwardingTraceParser::ForwardingTraceParser(TraceProcessorContext* context)
+ : context_(context) {}
+
+ForwardingTraceParser::~ForwardingTraceParser() {}
+
+util::Status ForwardingTraceParser::Parse(std::unique_ptr<uint8_t[]> data,
+ size_t size) {
+ // If this is the first Parse() call, guess the trace type and create the
+ // appropriate parser.
+
+ if (!reader_) {
+ TraceType trace_type;
+ {
+ auto scoped_trace = context_->storage->TraceExecutionTimeIntoStats(
+ stats::guess_trace_type_duration_ns);
+ trace_type = GuessTraceType(data.get(), size);
+ }
+ switch (trace_type) {
+ case kJsonTraceType: {
+ PERFETTO_DLOG("JSON trace detected");
+#if PERFETTO_BUILDFLAG(PERFETTO_STANDALONE_BUILD) || \
+ PERFETTO_BUILD_WITH_CHROMIUM
+ reader_.reset(new JsonTraceTokenizer(context_));
+ // JSON traces have no guarantees about the order of events in them.
+ int64_t window_size_ns = std::numeric_limits<int64_t>::max();
+ context_->sorter.reset(new TraceSorter(context_, window_size_ns));
+ context_->parser.reset(new JsonTraceParser(context_));
+#else
+ PERFETTO_FATAL("JSON traces not supported.");
+#endif
+ break;
+ }
+ case kProtoTraceType: {
+ PERFETTO_DLOG("Proto trace detected");
+ // This will be reduced once we read the trace config and we see flush
+ // period being set.
+ int64_t window_size_ns = std::numeric_limits<int64_t>::max();
+ reader_.reset(new ProtoTraceTokenizer(context_));
+ context_->sorter.reset(new TraceSorter(context_, window_size_ns));
+ context_->parser.reset(new ProtoTraceParser(context_));
+ break;
+ }
+ case kFuchsiaTraceType: {
+ PERFETTO_DLOG("Fuchsia trace detected");
+ // Fuschia traces can have massively out of order events.
+ int64_t window_size_ns = std::numeric_limits<int64_t>::max();
+ reader_.reset(new FuchsiaTraceTokenizer(context_));
+ context_->sorter.reset(new TraceSorter(context_, window_size_ns));
+ context_->parser.reset(new FuchsiaTraceParser(context_));
+ break;
+ }
+ case kSystraceTraceType:
+ PERFETTO_DLOG("Systrace trace detected");
+ reader_.reset(new SystraceTraceParser(context_));
+ break;
+ case kGzipTraceType:
+ PERFETTO_DLOG("gzip trace detected");
+ reader_.reset(new GzipTraceParser(context_));
+ break;
+ case kCtraceTraceType:
+ PERFETTO_DLOG("ctrace trace detected");
+ reader_.reset(new GzipTraceParser(context_));
+ break;
+ case kUnknownTraceType:
+ return util::ErrStatus("Unknown trace type provided");
+ }
+ }
+
+ return reader_->Parse(std::move(data), size);
+}
+
+TraceType GuessTraceType(const uint8_t* data, size_t size) {
+ if (size == 0)
+ return kUnknownTraceType;
+ std::string start(reinterpret_cast<const char*>(data),
+ std::min<size_t>(size, 20));
+ std::string start_minus_white_space = RemoveWhitespace(start);
+ if (base::StartsWith(start_minus_white_space, "{\"traceEvents\":["))
+ return kJsonTraceType;
+ if (base::StartsWith(start_minus_white_space, "[{"))
+ return kJsonTraceType;
+ if (size >= 8) {
+ uint64_t first_word = *reinterpret_cast<const uint64_t*>(data);
+ if (first_word == kFuchsiaMagicNumber)
+ return kFuchsiaTraceType;
+ }
+
+ // Systrace with header but no leading HTML.
+ if (base::StartsWith(start, "# tracer"))
+ return kSystraceTraceType;
+
+ // Systrace with leading HTML.
+ if (base::StartsWith(start, "<!DOCTYPE html>") ||
+ base::StartsWith(start, "<html>"))
+ return kSystraceTraceType;
+
+ // Systrace with no header or leading HTML.
+ if (base::StartsWith(start, " "))
+ return kSystraceTraceType;
+
+ // Ctrace is deflate'ed systrace.
+ if (base::StartsWith(start, "TRACE:"))
+ return kCtraceTraceType;
+
+ // gzip'ed trace containing one of the other formats.
+ if (base::StartsWith(start, "\x1f\x8b"))
+ return kGzipTraceType;
+
+ return kProtoTraceType;
+}
+
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/forwarding_trace_parser.h b/src/trace_processor/forwarding_trace_parser.h
new file mode 100644
index 0000000..f7216b3
--- /dev/null
+++ b/src/trace_processor/forwarding_trace_parser.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_FORWARDING_TRACE_PARSER_H_
+#define SRC_TRACE_PROCESSOR_FORWARDING_TRACE_PARSER_H_
+
+#include "src/trace_processor/chunked_trace_reader.h"
+
+#include "src/trace_processor/trace_processor_context.h"
+
+namespace perfetto {
+namespace trace_processor {
+
+enum TraceType {
+ kUnknownTraceType,
+ kProtoTraceType,
+ kJsonTraceType,
+ kFuchsiaTraceType,
+ kSystraceTraceType,
+ kGzipTraceType,
+ kCtraceTraceType,
+};
+
+TraceType GuessTraceType(const uint8_t* data, size_t size);
+
+class ForwardingTraceParser : public ChunkedTraceReader {
+ public:
+ explicit ForwardingTraceParser(TraceProcessorContext*);
+ ~ForwardingTraceParser() override;
+
+ // ChunkedTraceReader implementation
+ util::Status Parse(std::unique_ptr<uint8_t[]>, size_t) override;
+
+ private:
+ TraceProcessorContext* const context_;
+ std::unique_ptr<ChunkedTraceReader> reader_;
+};
+
+} // namespace trace_processor
+} // namespace perfetto
+
+#endif // SRC_TRACE_PROCESSOR_FORWARDING_TRACE_PARSER_H_
diff --git a/src/trace_processor/trace_processor_impl_unittest.cc b/src/trace_processor/forwarding_trace_parser_unittest.cc
similarity index 96%
rename from src/trace_processor/trace_processor_impl_unittest.cc
rename to src/trace_processor/forwarding_trace_parser_unittest.cc
index 24611d0..df4c013 100644
--- a/src/trace_processor/trace_processor_impl_unittest.cc
+++ b/src/trace_processor/forwarding_trace_parser_unittest.cc
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-#include "src/trace_processor/trace_processor_impl.h"
+#include "src/trace_processor/forwarding_trace_parser.h"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
diff --git a/src/trace_processor/gzip_trace_parser.cc b/src/trace_processor/gzip_trace_parser.cc
index 7b93af9..a374019 100644
--- a/src/trace_processor/gzip_trace_parser.cc
+++ b/src/trace_processor/gzip_trace_parser.cc
@@ -18,7 +18,8 @@
#include <zlib.h>
-#include "src/trace_processor/systrace_trace_parser.h"
+#include "perfetto/base/logging.h"
+#include "src/trace_processor/forwarding_trace_parser.h"
namespace perfetto {
namespace trace_processor {
@@ -28,7 +29,7 @@
z_stream_->zalloc = Z_NULL;
z_stream_->zfree = Z_NULL;
z_stream_->opaque = Z_NULL;
- inflateInit(z_stream_.get());
+ inflateInit2(z_stream_.get(), 32 + 15);
}
GzipTraceParser::~GzipTraceParser() {
@@ -41,13 +42,17 @@
uint8_t* start = data.get();
size_t len = size;
- static const char kSystraceFilerHeader[] = "TRACE:\n";
if (!inner_) {
- inner_.reset(new SystraceTraceParser(context_));
+ inner_.reset(new ForwardingTraceParser(context_));
- // Strip the header by ignoring the associated bytes.
- start += strlen(kSystraceFilerHeader);
- len -= strlen(kSystraceFilerHeader);
+ // .ctrace files begin with "TRACE:" strip this if present.
+ static const char kSystraceFileHeader[] = "TRACE:\n";
+ if (size >= strlen(kSystraceFileHeader) &&
+ strncmp(reinterpret_cast<char*>(start), kSystraceFileHeader,
+ strlen(kSystraceFileHeader)) == 0) {
+ start += strlen(kSystraceFileHeader);
+ len -= strlen(kSystraceFileHeader);
+ }
}
z_stream_->next_in = start;
diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc
index 710c770..70f18d2 100644
--- a/src/trace_processor/trace_processor_impl.cc
+++ b/src/trace_processor/trace_processor_impl.cc
@@ -33,9 +33,7 @@
#include "src/trace_processor/counter_definitions_table.h"
#include "src/trace_processor/counter_values_table.h"
#include "src/trace_processor/event_tracker.h"
-#include "src/trace_processor/fuchsia_trace_parser.h"
-#include "src/trace_processor/fuchsia_trace_tokenizer.h"
-#include "src/trace_processor/gzip_trace_parser.h"
+#include "src/trace_processor/forwarding_trace_parser.h"
#include "src/trace_processor/heap_profile_allocation_table.h"
#include "src/trace_processor/heap_profile_callsite_table.h"
#include "src/trace_processor/heap_profile_frame_table.h"
@@ -49,7 +47,6 @@
#include "src/trace_processor/metrics/sql_metrics.h"
#include "src/trace_processor/process_table.h"
#include "src/trace_processor/process_tracker.h"
-#include "src/trace_processor/proto_trace_parser.h"
#include "src/trace_processor/proto_trace_tokenizer.h"
#include "src/trace_processor/raw_table.h"
#include "src/trace_processor/sched_slice_table.h"
@@ -77,8 +74,6 @@
#if PERFETTO_BUILDFLAG(PERFETTO_STANDALONE_BUILD) || \
PERFETTO_BUILD_WITH_CHROMIUM
#include "src/trace_processor/export_json.h"
-#include "src/trace_processor/json_trace_parser.h"
-#include "src/trace_processor/json_trace_tokenizer.h"
#endif
// In Android and Chromium tree builds, we don't have the percentile module.
@@ -94,12 +89,6 @@
namespace trace_processor {
namespace {
-std::string RemoveWhitespace(const std::string& input) {
- std::string str(input);
- str.erase(std::remove_if(str.begin(), str.end(), ::isspace), str.end());
- return str;
-}
-
void InitializeSqlite(sqlite3* db) {
char* error = nullptr;
sqlite3_exec(db, "PRAGMA temp_store=2", 0, 0, &error);
@@ -265,48 +254,8 @@
}
}
-// Fuchsia traces have a magic number as documented here:
-// https://fuchsia.googlesource.com/fuchsia/+/HEAD/docs/development/tracing/trace-format/README.md#magic-number-record-trace-info-type-0
-constexpr uint64_t kFuchsiaMagicNumber = 0x0016547846040010;
-
} // namespace
-TraceType GuessTraceType(const uint8_t* data, size_t size) {
- if (size == 0)
- return kUnknownTraceType;
- std::string start(reinterpret_cast<const char*>(data),
- std::min<size_t>(size, 20));
- std::string start_minus_white_space = RemoveWhitespace(start);
- if (base::StartsWith(start_minus_white_space, "{\"traceEvents\":["))
- return kJsonTraceType;
- if (base::StartsWith(start_minus_white_space, "[{"))
- return kJsonTraceType;
- if (size >= 8) {
- uint64_t first_word = *reinterpret_cast<const uint64_t*>(data);
- if (first_word == kFuchsiaMagicNumber)
- return kFuchsiaTraceType;
- }
-
- // Systrace with header but no leading HTML.
- if (base::StartsWith(start, "# tracer"))
- return kSystraceTraceType;
-
- // Systrace with leading HTML.
- if (base::StartsWith(start, "<!DOCTYPE html>") ||
- base::StartsWith(start, "<html>"))
- return kSystraceTraceType;
-
- // Systrace with no header or leading HTML.
- if (base::StartsWith(start, " "))
- return kSystraceTraceType;
-
- // Ctrace is GZIPed systrace with no headers.
- if (base::StartsWith(start, "TRACE:"))
- return kCtraceTraceType;
-
- return kProtoTraceType;
-}
-
TraceProcessorImpl::TraceProcessorImpl(const Config& cfg) {
sqlite3* db = nullptr;
PERFETTO_CHECK(sqlite3_initialize() == SQLITE_OK);
@@ -368,58 +317,8 @@
if (unrecoverable_parse_error_)
return util::ErrStatus(
"Failed unrecoverably while parsing in a previous Parse call");
-
- // If this is the first Parse() call, guess the trace type and create the
- // appropriate parser.
- if (!context_.chunk_reader) {
- TraceType trace_type;
- {
- auto scoped_trace = context_.storage->TraceExecutionTimeIntoStats(
- stats::guess_trace_type_duration_ns);
- trace_type = GuessTraceType(data.get(), size);
- }
- switch (trace_type) {
- case kJsonTraceType: {
- PERFETTO_DLOG("Legacy JSON trace detected");
-#if PERFETTO_BUILDFLAG(PERFETTO_STANDALONE_BUILD) || \
- PERFETTO_BUILD_WITH_CHROMIUM
- context_.chunk_reader.reset(new JsonTraceTokenizer(&context_));
- // JSON traces have no guarantees about the order of events in them.
- int64_t window_size_ns = std::numeric_limits<int64_t>::max();
- context_.sorter.reset(new TraceSorter(&context_, window_size_ns));
- context_.parser.reset(new JsonTraceParser(&context_));
-#else
- PERFETTO_FATAL("JSON traces not supported.");
-#endif
- break;
- }
- case kProtoTraceType: {
- // This will be reduced once we read the trace config and we see flush
- // period being set.
- int64_t window_size_ns = std::numeric_limits<int64_t>::max();
- context_.chunk_reader.reset(new ProtoTraceTokenizer(&context_));
- context_.sorter.reset(new TraceSorter(&context_, window_size_ns));
- context_.parser.reset(new ProtoTraceParser(&context_));
- break;
- }
- case kFuchsiaTraceType: {
- // Fuschia traces can have massively out of order events.
- int64_t window_size_ns = std::numeric_limits<int64_t>::max();
- context_.chunk_reader.reset(new FuchsiaTraceTokenizer(&context_));
- context_.sorter.reset(new TraceSorter(&context_, window_size_ns));
- context_.parser.reset(new FuchsiaTraceParser(&context_));
- break;
- }
- case kSystraceTraceType:
- context_.chunk_reader.reset(new SystraceTraceParser(&context_));
- break;
- case kCtraceTraceType:
- context_.chunk_reader.reset(new GzipTraceParser(&context_));
- break;
- case kUnknownTraceType:
- return util::ErrStatus("Unknown trace type provided");
- }
- }
+ if (!context_.chunk_reader)
+ context_.chunk_reader.reset(new ForwardingTraceParser(&context_));
auto scoped_trace = context_.storage->TraceExecutionTimeIntoStats(
stats::parse_trace_duration_ns);
diff --git a/src/trace_processor/trace_processor_impl.h b/src/trace_processor/trace_processor_impl.h
index e9c3435..0ef2d7e 100644
--- a/src/trace_processor/trace_processor_impl.h
+++ b/src/trace_processor/trace_processor_impl.h
@@ -36,17 +36,6 @@
namespace trace_processor {
-enum TraceType {
- kUnknownTraceType,
- kProtoTraceType,
- kJsonTraceType,
- kFuchsiaTraceType,
- kSystraceTraceType,
- kCtraceTraceType,
-};
-
-TraceType GuessTraceType(const uint8_t* data, size_t size);
-
// Coordinates the loading of traces from an arbitrary source and allows
// execution of SQL queries on the events in these traces.
class TraceProcessorImpl : public TraceProcessor {
