libunwindstack/MemoryXz.cpp - platform/system/unwinding - Git at Google

 /*
  * Copyright (C) 2021 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>

 #include <algorithm>
 #include <atomic>
 #include <memory>
 #include <mutex>
 #include <string>
 #include <utility>

 #include <7zCrc.h>
 #include <Xz.h>
 #include <XzCrc64.h>

 #include <unwindstack/Log.h>

 #include "MemoryXz.h"

 namespace unwindstack {

 // Statistics (used only for optional debug log messages).
 static constexpr bool kLogMemoryXzUsage = false;
 std::atomic_size_t MemoryXz::total_used_ = 0;
 std::atomic_size_t MemoryXz::total_size_ = 0;
 std::atomic_size_t MemoryXz::total_open_ = 0;

 MemoryXz::MemoryXz(Memory* memory, uint64_t addr, uint64_t size, const std::string& name)
     : compressed_memory_(memory), compressed_addr_(addr), compressed_size_(size), name_(name) {
   total_open_ += 1;
 }

 bool MemoryXz::Init() {
   static std::once_flag crc_initialized;
   std::call_once(crc_initialized, []() {
     CrcGenerateTable();
     Crc64GenerateTable();
   });
   if (compressed_size_ >= kMaxCompressedSize) {
     return false;
   }
   if (!ReadBlocks()) {
     return false;
   }

   // All blocks (except the last one) must have the same power-of-2 size.
   if (blocks_.size() > 1) {
     size_t block_size_log2 = __builtin_ctz(blocks_.front().decompressed_size);
     auto correct_size = [=](XzBlock& b) { return b.decompressed_size == (1 << block_size_log2); };
     if (std::all_of(blocks_.begin(), std::prev(blocks_.end()), correct_size) &&
         blocks_.back().decompressed_size <= (1 << block_size_log2)) {
       block_size_log2_ = block_size_log2;
     } else {
       // Inconsistent block-sizes.  Decompress and merge everything now.
       std::unique_ptr<uint8_t[]> data(new uint8_t[size_]);
       size_t offset = 0;
       for (XzBlock& block : blocks_) {
         if (!Decompress(&block)) {
           return false;
         }
         memcpy(data.get() + offset, block.decompressed_data.get(), block.decompressed_size);
         offset += block.decompressed_size;
       }
       blocks_.clear();
       blocks_.push_back(XzBlock{
           .decompressed_data = std::move(data),
           .decompressed_size = size_,
       });
       block_size_log2_ = 31;  // Because 32 bits is too big (shift right by 32 is not allowed).
     }
   }

   return true;
 }

 MemoryXz::~MemoryXz() {
   total_used_ -= used_;
   total_size_ -= size_;
   total_open_ -= 1;
 }

 size_t MemoryXz::Read(uint64_t addr, void* buffer, size_t size) {
   if (addr >= size_) {
     return 0;  // Read past the end.
   }
   uint8_t* dst = reinterpret_cast<uint8_t*>(buffer);  // Position in the output buffer.
   for (size_t i = addr >> block_size_log2_; i < blocks_.size(); i++) {
     XzBlock* block = &blocks_[i];
     if (block->decompressed_data == nullptr) {
       if (!Decompress(block)) {
         break;
       }
     }
     size_t offset = (addr - (i << block_size_log2_));  // Start inside the block.
     size_t copy_bytes = std::min<size_t>(size, block->decompressed_size - offset);
     memcpy(dst, block->decompressed_data.get() + offset, copy_bytes);
     dst += copy_bytes;
     addr += copy_bytes;
     size -= copy_bytes;
     if (size == 0) {
       break;
     }
   }
   return dst - reinterpret_cast<uint8_t*>(buffer);
 }

 bool MemoryXz::ReadBlocks() {
   static ISzAlloc alloc;
   alloc.Alloc = [](ISzAllocPtr, size_t size) { return malloc(size); };
   alloc.Free = [](ISzAllocPtr, void* ptr) { return free(ptr); };

   // Read the compressed data, so we can quickly scan through the headers.
   std::unique_ptr<uint8_t[]> compressed_data(new (std::nothrow) uint8_t[compressed_size_]);
   if (compressed_data.get() == nullptr) {
     return false;
   }
   if (!compressed_memory_->ReadFully(compressed_addr_, compressed_data.get(), compressed_size_)) {
     return false;
   }

   // Implement the required interface for communication
   // (written in C so we can not use virtual methods or member functions).
   struct XzLookInStream : public ILookInStream, public ICompressProgress {
     static SRes LookImpl(const ILookInStream* p, const void** buf, size_t* size) {
       auto* ctx = reinterpret_cast<const XzLookInStream*>(p);
       *buf = ctx->data + ctx->offset;
       *size = std::min(*size, ctx->size - ctx->offset);
       return SZ_OK;
     }
     static SRes SkipImpl(const ILookInStream* p, size_t len) {
       auto* ctx = reinterpret_cast<XzLookInStream*>(const_cast<ILookInStream*>(p));
       ctx->offset += len;
       return SZ_OK;
     }
     static SRes ReadImpl(const ILookInStream* p, void* buf, size_t* size) {
       auto* ctx = reinterpret_cast<const XzLookInStream*>(p);
       *size = std::min(*size, ctx->size - ctx->offset);
       memcpy(buf, ctx->data + ctx->offset, *size);
       return SZ_OK;
     }
     static SRes SeekImpl(const ILookInStream* p, Int64* pos, ESzSeek origin) {
       auto* ctx = reinterpret_cast<XzLookInStream*>(const_cast<ILookInStream*>(p));
       switch (origin) {
         case SZ_SEEK_SET:
           ctx->offset = *pos;
           break;
         case SZ_SEEK_CUR:
           ctx->offset += *pos;
           break;
         case SZ_SEEK_END:
           ctx->offset = ctx->size + *pos;
           break;
       }
       *pos = ctx->offset;
       return SZ_OK;
     }
     static SRes ProgressImpl(const ICompressProgress*, UInt64, UInt64) { return SZ_OK; }
     size_t offset;
     uint8_t* data;
     size_t size;
   };
   XzLookInStream callbacks;
   callbacks.Look = &XzLookInStream::LookImpl;
   callbacks.Skip = &XzLookInStream::SkipImpl;
   callbacks.Read = &XzLookInStream::ReadImpl;
   callbacks.Seek = &XzLookInStream::SeekImpl;
   callbacks.Progress = &XzLookInStream::ProgressImpl;
   callbacks.offset = 0;
   callbacks.data = compressed_data.get();
   callbacks.size = compressed_size_;

   // Iterate over the internal XZ blocks without decompressing them.
   CXzs xzs;
   Xzs_Construct(&xzs);
   Int64 end_offset = compressed_size_;
   if (Xzs_ReadBackward(&xzs, &callbacks, &end_offset, &callbacks, &alloc) == SZ_OK) {
     blocks_.reserve(Xzs_GetNumBlocks(&xzs));
     size_t dst_offset = 0;
     for (int s = xzs.num - 1; s >= 0; s--) {
       const CXzStream& stream = xzs.streams[s];
       size_t src_offset = stream.startOffset + XZ_STREAM_HEADER_SIZE;
       for (size_t b = 0; b < stream.numBlocks; b++) {
         const CXzBlockSizes& block = stream.blocks[b];
         blocks_.push_back(XzBlock{
             .decompressed_data = nullptr,  // Lazy allocation and decompression.
             .decompressed_size = static_cast<uint32_t>(block.unpackSize),
             .compressed_offset = static_cast<uint32_t>(src_offset),
             .compressed_size = static_cast<uint32_t>((block.totalSize + 3) & ~3u),
             .stream_flags = stream.flags,
         });
         dst_offset += blocks_.back().decompressed_size;
         src_offset += blocks_.back().compressed_size;
       }
     }
     size_ = dst_offset;
     total_size_ += dst_offset;
   }
   Xzs_Free(&xzs, &alloc);
   return !blocks_.empty();
 }

 bool MemoryXz::Decompress(XzBlock* block) {
   static ISzAlloc alloc;
   alloc.Alloc = [](ISzAllocPtr, size_t size) { return malloc(size); };
   alloc.Free = [](ISzAllocPtr, void* ptr) { return free(ptr); };

   // Read the compressed data for this block.
   std::unique_ptr<uint8_t[]> compressed_data(new (std::nothrow) uint8_t[block->compressed_size]);
   if (compressed_data.get() == nullptr) {
     return false;
   }
   if (!compressed_memory_->ReadFully(compressed_addr_ + block->compressed_offset,
                                      compressed_data.get(), block->compressed_size)) {
     return false;
   }

   // Allocate decompressed memory.
   std::unique_ptr<uint8_t[]> decompressed_data(new uint8_t[block->decompressed_size]);
   if (decompressed_data == nullptr) {
     return false;
   }

   // Decompress.
   CXzUnpacker state{};
   XzUnpacker_Construct(&state, &alloc);
   state.streamFlags = block->stream_flags;
   XzUnpacker_PrepareToRandomBlockDecoding(&state);
   size_t decompressed_size = block->decompressed_size;
   size_t compressed_size = block->compressed_size;
   ECoderStatus status;
   XzUnpacker_SetOutBuf(&state, decompressed_data.get(), decompressed_size);
   int return_val =
       XzUnpacker_Code(&state, /*decompressed_data=*/nullptr, &decompressed_size,
                       compressed_data.get(), &compressed_size, true, CODER_FINISH_END, &status);
   XzUnpacker_Free(&state);
   if (return_val != SZ_OK || status != CODER_STATUS_FINISHED_WITH_MARK) {
     Log::Error("Cannot decompress \"%s\"", name_.c_str());
     return false;
   }

   used_ += block->decompressed_size;
   total_used_ += block->decompressed_size;
   if (kLogMemoryXzUsage) {
     Log::Info("decompressed memory: %zi%% of %ziKB (%zi files), %i%% of %iKB (%s)",
               100 * total_used_ / total_size_, total_size_ / 1024, total_open_.load(),
               100 * used_ / size_, size_ / 1024, name_.c_str());
   }

   block->decompressed_data = std::move(decompressed_data);
   return true;
 }

 }  // namespace unwindstack
	/*
	* Copyright (C) 2021 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include <stdint.h>
	#include <stdlib.h>
	#include <string.h>

	#include <algorithm>
	#include <atomic>
	#include <memory>
	#include <mutex>
	#include <string>
	#include <utility>

	#include <7zCrc.h>
	#include <Xz.h>
	#include <XzCrc64.h>

	#include <unwindstack/Log.h>

	#include "MemoryXz.h"

	namespace unwindstack {

	// Statistics (used only for optional debug log messages).
	static constexpr bool kLogMemoryXzUsage = false;
	std::atomic_size_t MemoryXz::total_used_ = 0;
	std::atomic_size_t MemoryXz::total_size_ = 0;
	std::atomic_size_t MemoryXz::total_open_ = 0;

	MemoryXz::MemoryXz(Memory* memory, uint64_t addr, uint64_t size, const std::string& name)
	: compressed_memory_(memory), compressed_addr_(addr), compressed_size_(size), name_(name) {
	total_open_ += 1;
	}

	bool MemoryXz::Init() {
	static std::once_flag crc_initialized;
	std::call_once(crc_initialized, []() {
	CrcGenerateTable();
	Crc64GenerateTable();
	});
	if (compressed_size_ >= kMaxCompressedSize) {
	return false;
	}
	if (!ReadBlocks()) {
	return false;
	}

	// All blocks (except the last one) must have the same power-of-2 size.
	if (blocks_.size() > 1) {
	size_t block_size_log2 = __builtin_ctz(blocks_.front().decompressed_size);
	auto correct_size = [=](XzBlock& b) { return b.decompressed_size == (1 << block_size_log2); };
	if (std::all_of(blocks_.begin(), std::prev(blocks_.end()), correct_size) &&
	blocks_.back().decompressed_size <= (1 << block_size_log2)) {
	block_size_log2_ = block_size_log2;
	} else {
	// Inconsistent block-sizes. Decompress and merge everything now.
	std::unique_ptr<uint8_t[]> data(new uint8_t[size_]);
	size_t offset = 0;
	for (XzBlock& block : blocks_) {
	if (!Decompress(&block)) {
	return false;
	}
	memcpy(data.get() + offset, block.decompressed_data.get(), block.decompressed_size);
	offset += block.decompressed_size;
	}
	blocks_.clear();
	blocks_.push_back(XzBlock{
	.decompressed_data = std::move(data),
	.decompressed_size = size_,
	});
	block_size_log2_ = 31; // Because 32 bits is too big (shift right by 32 is not allowed).
	}
	}

	return true;
	}

	MemoryXz::~MemoryXz() {
	total_used_ -= used_;
	total_size_ -= size_;
	total_open_ -= 1;
	}

	size_t MemoryXz::Read(uint64_t addr, void* buffer, size_t size) {
	if (addr >= size_) {
	return 0; // Read past the end.
	}
	uint8_t* dst = reinterpret_cast<uint8_t*>(buffer); // Position in the output buffer.
	for (size_t i = addr >> block_size_log2_; i < blocks_.size(); i++) {
	XzBlock* block = &blocks_[i];
	if (block->decompressed_data == nullptr) {
	if (!Decompress(block)) {
	break;
	}
	}
	size_t offset = (addr - (i << block_size_log2_)); // Start inside the block.
	size_t copy_bytes = std::min<size_t>(size, block->decompressed_size - offset);
	memcpy(dst, block->decompressed_data.get() + offset, copy_bytes);
	dst += copy_bytes;
	addr += copy_bytes;
	size -= copy_bytes;
	if (size == 0) {
	break;
	}
	}
	return dst - reinterpret_cast<uint8_t*>(buffer);
	}

	bool MemoryXz::ReadBlocks() {
	static ISzAlloc alloc;
	alloc.Alloc = [](ISzAllocPtr, size_t size) { return malloc(size); };
	alloc.Free = [](ISzAllocPtr, void* ptr) { return free(ptr); };

	// Read the compressed data, so we can quickly scan through the headers.
	std::unique_ptr<uint8_t[]> compressed_data(new (std::nothrow) uint8_t[compressed_size_]);
	if (compressed_data.get() == nullptr) {
	return false;
	}
	if (!compressed_memory_->ReadFully(compressed_addr_, compressed_data.get(), compressed_size_)) {
	return false;
	}

	// Implement the required interface for communication
	// (written in C so we can not use virtual methods or member functions).
	struct XzLookInStream : public ILookInStream, public ICompressProgress {
	static SRes LookImpl(const ILookInStream* p, const void** buf, size_t* size) {
	auto* ctx = reinterpret_cast<const XzLookInStream*>(p);
	*buf = ctx->data + ctx->offset;
	size = std::min(size, ctx->size - ctx->offset);
	return SZ_OK;
	}
	static SRes SkipImpl(const ILookInStream* p, size_t len) {
	auto* ctx = reinterpret_cast<XzLookInStream>(const_cast<ILookInStream>(p));
	ctx->offset += len;
	return SZ_OK;
	}
	static SRes ReadImpl(const ILookInStream* p, void* buf, size_t* size) {
	auto* ctx = reinterpret_cast<const XzLookInStream*>(p);
	size = std::min(size, ctx->size - ctx->offset);
	memcpy(buf, ctx->data + ctx->offset, *size);
	return SZ_OK;
	}
	static SRes SeekImpl(const ILookInStream* p, Int64* pos, ESzSeek origin) {
	auto* ctx = reinterpret_cast<XzLookInStream>(const_cast<ILookInStream>(p));
	switch (origin) {
	case SZ_SEEK_SET:
	ctx->offset = *pos;
	break;
	case SZ_SEEK_CUR:
	ctx->offset += *pos;
	break;
	case SZ_SEEK_END:
	ctx->offset = ctx->size + *pos;
	break;
	}
	*pos = ctx->offset;
	return SZ_OK;
	}
	static SRes ProgressImpl(const ICompressProgress*, UInt64, UInt64) { return SZ_OK; }
	size_t offset;
	uint8_t* data;
	size_t size;
	};
	XzLookInStream callbacks;
	callbacks.Look = &XzLookInStream::LookImpl;
	callbacks.Skip = &XzLookInStream::SkipImpl;
	callbacks.Read = &XzLookInStream::ReadImpl;
	callbacks.Seek = &XzLookInStream::SeekImpl;
	callbacks.Progress = &XzLookInStream::ProgressImpl;
	callbacks.offset = 0;
	callbacks.data = compressed_data.get();
	callbacks.size = compressed_size_;

	// Iterate over the internal XZ blocks without decompressing them.
	CXzs xzs;
	Xzs_Construct(&xzs);
	Int64 end_offset = compressed_size_;
	if (Xzs_ReadBackward(&xzs, &callbacks, &end_offset, &callbacks, &alloc) == SZ_OK) {
	blocks_.reserve(Xzs_GetNumBlocks(&xzs));
	size_t dst_offset = 0;
	for (int s = xzs.num - 1; s >= 0; s--) {
	const CXzStream& stream = xzs.streams[s];
	size_t src_offset = stream.startOffset + XZ_STREAM_HEADER_SIZE;
	for (size_t b = 0; b < stream.numBlocks; b++) {
	const CXzBlockSizes& block = stream.blocks[b];
	blocks_.push_back(XzBlock{
	.decompressed_data = nullptr, // Lazy allocation and decompression.
	.decompressed_size = static_cast<uint32_t>(block.unpackSize),
	.compressed_offset = static_cast<uint32_t>(src_offset),
	.compressed_size = static_cast<uint32_t>((block.totalSize + 3) & ~3u),
	.stream_flags = stream.flags,
	});
	dst_offset += blocks_.back().decompressed_size;
	src_offset += blocks_.back().compressed_size;
	}
	}
	size_ = dst_offset;
	total_size_ += dst_offset;
	}
	Xzs_Free(&xzs, &alloc);
	return !blocks_.empty();
	}

	bool MemoryXz::Decompress(XzBlock* block) {
	static ISzAlloc alloc;
	alloc.Alloc = [](ISzAllocPtr, size_t size) { return malloc(size); };
	alloc.Free = [](ISzAllocPtr, void* ptr) { return free(ptr); };

	// Read the compressed data for this block.
	std::unique_ptr<uint8_t[]> compressed_data(new (std::nothrow) uint8_t[block->compressed_size]);
	if (compressed_data.get() == nullptr) {
	return false;
	}
	if (!compressed_memory_->ReadFully(compressed_addr_ + block->compressed_offset,
	compressed_data.get(), block->compressed_size)) {
	return false;
	}

	// Allocate decompressed memory.
	std::unique_ptr<uint8_t[]> decompressed_data(new uint8_t[block->decompressed_size]);
	if (decompressed_data == nullptr) {
	return false;
	}

	// Decompress.
	CXzUnpacker state{};
	XzUnpacker_Construct(&state, &alloc);
	state.streamFlags = block->stream_flags;
	XzUnpacker_PrepareToRandomBlockDecoding(&state);
	size_t decompressed_size = block->decompressed_size;
	size_t compressed_size = block->compressed_size;
	ECoderStatus status;
	XzUnpacker_SetOutBuf(&state, decompressed_data.get(), decompressed_size);
	int return_val =
	XzUnpacker_Code(&state, /decompressed_data=/nullptr, &decompressed_size,
	compressed_data.get(), &compressed_size, true, CODER_FINISH_END, &status);
	XzUnpacker_Free(&state);
	if (return_val != SZ_OK \|\| status != CODER_STATUS_FINISHED_WITH_MARK) {
	Log::Error("Cannot decompress \"%s\"", name_.c_str());
	return false;
	}

	used_ += block->decompressed_size;
	total_used_ += block->decompressed_size;
	if (kLogMemoryXzUsage) {
	Log::Info("decompressed memory: %zi%% of %ziKB (%zi files), %i%% of %iKB (%s)",
	100 * total_used_ / total_size_, total_size_ / 1024, total_open_.load(),
	100 * used_ / size_, size_ / 1024, name_.c_str());
	}

	block->decompressed_data = std::move(decompressed_data);
	return true;
	}

	} // namespace unwindstack