diff --git a/simpleperf/build_id.h b/simpleperf/build_id.h
index a244e7f..5d587f2 100644
--- a/simpleperf/build_id.h
+++ b/simpleperf/build_id.h
@@ -53,6 +53,11 @@
     return memcmp(data_, build_id.data_, BUILD_ID_SIZE) == 0;
   }
 
+  bool IsEmpty() const {
+    static BuildId empty_build_id;
+    return *this == empty_build_id;
+  }
+
  private:
   unsigned char data_[BUILD_ID_SIZE];
 };
diff --git a/simpleperf/dso.cpp b/simpleperf/dso.cpp
index 2b63641..ca45c2d 100644
--- a/simpleperf/dso.cpp
+++ b/simpleperf/dso.cpp
@@ -162,18 +162,21 @@
     ParseSymbolsFromElfFile(vmlinux_, build_id,
                             std::bind(VmlinuxSymbolCallback, std::placeholders::_1, dso));
   } else {
-    BuildId real_build_id;
-    GetKernelBuildId(&real_build_id);
-    bool match = (build_id == real_build_id);
-    LOG(DEBUG) << "check kernel build id (" << (match ? "match" : "mismatch") << "): expected "
-               << build_id.ToString() << ", real " << real_build_id.ToString();
-    if (match) {
-      ProcessKernelSymbols("/proc/kallsyms",
-                           std::bind(&KernelSymbolCallback, std::placeholders::_1, dso));
+    if (!build_id.IsEmpty()) {
+      BuildId real_build_id;
+      GetKernelBuildId(&real_build_id);
+      bool match = (build_id == real_build_id);
+      LOG(DEBUG) << "check kernel build id (" << (match ? "match" : "mismatch") << "): expected "
+                 << build_id.ToString() << ", real " << real_build_id.ToString();
+      if (!match) {
+        return false;
+      }
     }
+    ProcessKernelSymbols("/proc/kallsyms",
+                         std::bind(&KernelSymbolCallback, std::placeholders::_1, dso));
   }
   FixupSymbolLength(dso);
-  return dso;
+  return true;
 }
 
 static void ParseSymbolCallback(const ElfFileSymbol& elf_symbol, DsoEntry* dso,
@@ -199,7 +202,7 @@
       symfs_dir_ + dso->path, build_id,
       std::bind(ParseSymbolCallback, std::placeholders::_1, dso, SymbolFilterForKernelModule));
   FixupSymbolLength(dso);
-  return dso;
+  return true;
 }
 
 static bool SymbolFilterForDso(const ElfFileSymbol& elf_symbol) {
@@ -240,7 +243,7 @@
     }
   }
   FixupSymbolLength(dso);
-  return dso;
+  return true;
 }
 
 BuildId DsoFactory::GetExpectedBuildId(const std::string& filename) {
diff --git a/simpleperf/read_elf.cpp b/simpleperf/read_elf.cpp
index 41120fe..028af72 100644
--- a/simpleperf/read_elf.cpp
+++ b/simpleperf/read_elf.cpp
@@ -209,14 +209,16 @@
     LOG(DEBUG) << filename << " is not an object file";
     return false;
   }
-  BuildId real_build_id;
-  GetBuildIdFromObjectFile(obj, &real_build_id);
-  bool result = (expected_build_id == real_build_id);
-  LOG(DEBUG) << "check build id for \"" << filename << "\" (" << (result ? "match" : "mismatch")
-             << "): expected " << expected_build_id.ToString() << ", real "
-             << real_build_id.ToString();
-  if (!result) {
-    return result;
+  if (!expected_build_id.IsEmpty()) {
+    BuildId real_build_id;
+    GetBuildIdFromObjectFile(obj, &real_build_id);
+    bool result = (expected_build_id == real_build_id);
+    LOG(DEBUG) << "check build id for \"" << filename << "\" (" << (result ? "match" : "mismatch")
+               << "): expected " << expected_build_id.ToString() << ", real "
+               << real_build_id.ToString();
+    if (!result) {
+      return false;
+    }
   }
 
   if (auto elf = llvm::dyn_cast<llvm::object::ELF32LEObjectFile>(obj)) {