| From 57a3cf95b276946559f9e044c7352c11303bb9c1 Mon Sep 17 00:00:00 2001 |
| From: Sean Purcell <me@seanp.xyz> |
| Date: Thu, 3 Aug 2017 17:47:03 -0700 |
| Subject: [PATCH v6] squashfs-tools: Add zstd support |
| |
| This patch adds zstd support to squashfs-tools. It works with zstd |
| versions >= 1.0.0. It was originally written by Sean Purcell. |
| |
| Signed-off-by: Sean Purcell <me@seanp.xyz> |
| Signed-off-by: Nick Terrell <terrelln@fb.com> |
| --- |
| v4 -> v5: |
| - Fix patch documentation to reflect that Sean Purcell is the author |
| - Don't strip trailing whitespace of unrelated code |
| - Make zstd_display_options() static |
| |
| v5 -> v6: |
| - Fix build instructions in Makefile |
| |
| squashfs-tools/Makefile | 20 ++++ |
| squashfs-tools/compressor.c | 8 ++ |
| squashfs-tools/squashfs_fs.h | 1 + |
| squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++ |
| squashfs-tools/zstd_wrapper.h | 48 ++++++++ |
| 5 files changed, 331 insertions(+) |
| create mode 100644 squashfs-tools/zstd_wrapper.c |
| create mode 100644 squashfs-tools/zstd_wrapper.h |
| |
| diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile |
| index 52d2582..22fc559 100644 |
| --- a/squashfs-tools/Makefile |
| +++ b/squashfs-tools/Makefile |
| @@ -75,6 +75,18 @@ GZIP_SUPPORT = 1 |
| #LZMA_SUPPORT = 1 |
| #LZMA_DIR = ../../../../LZMA/lzma465 |
| |
| + |
| +########### Building ZSTD support ############ |
| +# |
| +# The ZSTD library is supported |
| +# ZSTD homepage: http://zstd.net |
| +# ZSTD source repository: https://github.com/facebook/zstd |
| +# |
| +# To build using the ZSTD library - install the library and uncomment the |
| +# ZSTD_SUPPORT line below. |
| +# |
| +#ZSTD_SUPPORT = 1 |
| + |
| ######## Specifying default compression ######## |
| # |
| # The next line specifies which compression algorithm is used by default |
| @@ -177,6 +189,14 @@ LIBS += -llz4 |
| COMPRESSORS += lz4 |
| endif |
| |
| +ifeq ($(ZSTD_SUPPORT),1) |
| +CFLAGS += -DZSTD_SUPPORT |
| +MKSQUASHFS_OBJS += zstd_wrapper.o |
| +UNSQUASHFS_OBJS += zstd_wrapper.o |
| +LIBS += -lzstd |
| +COMPRESSORS += zstd |
| +endif |
| + |
| ifeq ($(XATTR_SUPPORT),1) |
| ifeq ($(XATTR_DEFAULT),1) |
| CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT |
| diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c |
| index 525e316..02b5e90 100644 |
| --- a/squashfs-tools/compressor.c |
| +++ b/squashfs-tools/compressor.c |
| @@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = { |
| extern struct compressor xz_comp_ops; |
| #endif |
| |
| +#ifndef ZSTD_SUPPORT |
| +static struct compressor zstd_comp_ops = { |
| + ZSTD_COMPRESSION, "zstd" |
| +}; |
| +#else |
| +extern struct compressor zstd_comp_ops; |
| +#endif |
| |
| static struct compressor unknown_comp_ops = { |
| 0, "unknown" |
| @@ -77,6 +84,7 @@ struct compressor *compressor[] = { |
| &lzo_comp_ops, |
| &lz4_comp_ops, |
| &xz_comp_ops, |
| + &zstd_comp_ops, |
| &unknown_comp_ops |
| }; |
| |
| diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h |
| index 791fe12..afca918 100644 |
| --- a/squashfs-tools/squashfs_fs.h |
| +++ b/squashfs-tools/squashfs_fs.h |
| @@ -277,6 +277,7 @@ typedef long long squashfs_inode; |
| #define LZO_COMPRESSION 3 |
| #define XZ_COMPRESSION 4 |
| #define LZ4_COMPRESSION 5 |
| +#define ZSTD_COMPRESSION 6 |
| |
| struct squashfs_super_block { |
| unsigned int s_magic; |
| diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c |
| new file mode 100644 |
| index 0000000..dcab75a |
| --- /dev/null |
| +++ b/squashfs-tools/zstd_wrapper.c |
| @@ -0,0 +1,254 @@ |
| +/* |
| + * Copyright (c) 2017 |
| + * Phillip Lougher <phillip@squashfs.org.uk> |
| + * |
| + * This program is free software; you can redistribute it and/or |
| + * modify it under the terms of the GNU General Public License |
| + * as published by the Free Software Foundation; either version 2, |
| + * or (at your option) any later version. |
| + * |
| + * This program is distributed in the hope that it will be useful, |
| + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| + * GNU General Public License for more details. |
| + * |
| + * zstd_wrapper.c |
| + * |
| + * Support for ZSTD compression http://zstd.net |
| + */ |
| + |
| +#include <stdio.h> |
| +#include <string.h> |
| +#include <stdlib.h> |
| +#include <zstd.h> |
| +#include <zstd_errors.h> |
| + |
| +#include "squashfs_fs.h" |
| +#include "zstd_wrapper.h" |
| +#include "compressor.h" |
| + |
| +static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; |
| + |
| +/* |
| + * This function is called by the options parsing code in mksquashfs.c |
| + * to parse any -X compressor option. |
| + * |
| + * This function returns: |
| + * >=0 (number of additional args parsed) on success |
| + * -1 if the option was unrecognised, or |
| + * -2 if the option was recognised, but otherwise bad in |
| + * some way (e.g. invalid parameter) |
| + * |
| + * Note: this function sets internal compressor state, but does not |
| + * pass back the results of the parsing other than success/failure. |
| + * The zstd_dump_options() function is called later to get the options in |
| + * a format suitable for writing to the filesystem. |
| + */ |
| +static int zstd_options(char *argv[], int argc) |
| +{ |
| + if (strcmp(argv[0], "-Xcompression-level") == 0) { |
| + if (argc < 2) { |
| + fprintf(stderr, "zstd: -Xcompression-level missing " |
| + "compression level\n"); |
| + fprintf(stderr, "zstd: -Xcompression-level it should " |
| + "be 1 <= n <= %d\n", ZSTD_maxCLevel()); |
| + goto failed; |
| + } |
| + |
| + compression_level = atoi(argv[1]); |
| + if (compression_level < 1 || |
| + compression_level > ZSTD_maxCLevel()) { |
| + fprintf(stderr, "zstd: -Xcompression-level invalid, it " |
| + "should be 1 <= n <= %d\n", ZSTD_maxCLevel()); |
| + goto failed; |
| + } |
| + |
| + return 1; |
| + } |
| + |
| + return -1; |
| +failed: |
| + return -2; |
| +} |
| + |
| +/* |
| + * This function is called by mksquashfs to dump the parsed |
| + * compressor options in a format suitable for writing to the |
| + * compressor options field in the filesystem (stored immediately |
| + * after the superblock). |
| + * |
| + * This function returns a pointer to the compression options structure |
| + * to be stored (and the size), or NULL if there are no compression |
| + * options. |
| + */ |
| +static void *zstd_dump_options(int block_size, int *size) |
| +{ |
| + static struct zstd_comp_opts comp_opts; |
| + |
| + /* don't return anything if the options are all default */ |
| + if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL) |
| + return NULL; |
| + |
| + comp_opts.compression_level = compression_level; |
| + |
| + SQUASHFS_INSWAP_COMP_OPTS(&comp_opts); |
| + |
| + *size = sizeof(comp_opts); |
| + return &comp_opts; |
| +} |
| + |
| +/* |
| + * This function is a helper specifically for the append mode of |
| + * mksquashfs. Its purpose is to set the internal compressor state |
| + * to the stored compressor options in the passed compressor options |
| + * structure. |
| + * |
| + * In effect this function sets up the compressor options |
| + * to the same state they were when the filesystem was originally |
| + * generated, this is to ensure on appending, the compressor uses |
| + * the same compression options that were used to generate the |
| + * original filesystem. |
| + * |
| + * Note, even if there are no compressor options, this function is still |
| + * called with an empty compressor structure (size == 0), to explicitly |
| + * set the default options, this is to ensure any user supplied |
| + * -X options on the appending mksquashfs command line are over-ridden. |
| + * |
| + * This function returns 0 on successful extraction of options, and -1 on error. |
| + */ |
| +static int zstd_extract_options(int block_size, void *buffer, int size) |
| +{ |
| + struct zstd_comp_opts *comp_opts = buffer; |
| + |
| + if (size == 0) { |
| + /* Set default values */ |
| + compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; |
| + return 0; |
| + } |
| + |
| + /* we expect a comp_opts structure of sufficient size to be present */ |
| + if (size < sizeof(*comp_opts)) |
| + goto failed; |
| + |
| + SQUASHFS_INSWAP_COMP_OPTS(comp_opts); |
| + |
| + if (comp_opts->compression_level < 1 || |
| + comp_opts->compression_level > ZSTD_maxCLevel()) { |
| + fprintf(stderr, "zstd: bad compression level in compression " |
| + "options structure\n"); |
| + goto failed; |
| + } |
| + |
| + compression_level = comp_opts->compression_level; |
| + |
| + return 0; |
| + |
| +failed: |
| + fprintf(stderr, "zstd: error reading stored compressor options from " |
| + "filesystem!\n"); |
| + |
| + return -1; |
| +} |
| + |
| +static void zstd_display_options(void *buffer, int size) |
| +{ |
| + struct zstd_comp_opts *comp_opts = buffer; |
| + |
| + /* we expect a comp_opts structure of sufficient size to be present */ |
| + if (size < sizeof(*comp_opts)) |
| + goto failed; |
| + |
| + SQUASHFS_INSWAP_COMP_OPTS(comp_opts); |
| + |
| + if (comp_opts->compression_level < 1 || |
| + comp_opts->compression_level > ZSTD_maxCLevel()) { |
| + fprintf(stderr, "zstd: bad compression level in compression " |
| + "options structure\n"); |
| + goto failed; |
| + } |
| + |
| + printf("\tcompression-level %d\n", comp_opts->compression_level); |
| + |
| + return; |
| + |
| +failed: |
| + fprintf(stderr, "zstd: error reading stored compressor options from " |
| + "filesystem!\n"); |
| +} |
| + |
| +/* |
| + * This function is called by mksquashfs to initialise the |
| + * compressor, before compress() is called. |
| + * |
| + * This function returns 0 on success, and -1 on error. |
| + */ |
| +static int zstd_init(void **strm, int block_size, int datablock) |
| +{ |
| + ZSTD_CCtx *cctx = ZSTD_createCCtx(); |
| + |
| + if (!cctx) { |
| + fprintf(stderr, "zstd: failed to allocate compression " |
| + "context!\n"); |
| + return -1; |
| + } |
| + |
| + *strm = cctx; |
| + return 0; |
| +} |
| + |
| +static int zstd_compress(void *strm, void *dest, void *src, int size, |
| + int block_size, int *error) |
| +{ |
| + const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size, |
| + src, size, compression_level); |
| + |
| + if (ZSTD_isError(res)) { |
| + /* FIXME: |
| + * zstd does not expose stable error codes. The error enum may |
| + * change between versions. Until upstream zstd stablizes the |
| + * error codes, we have no way of knowing why the error occurs. |
| + * zstd shouldn't fail to compress any input unless there isn't |
| + * enough output space. We assume that is the cause and return |
| + * the special error code for not enough output space. |
| + */ |
| + return 0; |
| + } |
| + |
| + return (int)res; |
| +} |
| + |
| +static int zstd_uncompress(void *dest, void *src, int size, int outsize, |
| + int *error) |
| +{ |
| + const size_t res = ZSTD_decompress(dest, outsize, src, size); |
| + |
| + if (ZSTD_isError(res)) { |
| + fprintf(stderr, "\t%d %d\n", outsize, size); |
| + |
| + *error = (int)ZSTD_getErrorCode(res); |
| + return -1; |
| + } |
| + |
| + return (int)res; |
| +} |
| + |
| +static void zstd_usage(void) |
| +{ |
| + fprintf(stderr, "\t -Xcompression-level <compression-level>\n"); |
| + fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default " |
| + "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL); |
| +} |
| + |
| +struct compressor zstd_comp_ops = { |
| + .init = zstd_init, |
| + .compress = zstd_compress, |
| + .uncompress = zstd_uncompress, |
| + .options = zstd_options, |
| + .dump_options = zstd_dump_options, |
| + .extract_options = zstd_extract_options, |
| + .display_options = zstd_display_options, |
| + .usage = zstd_usage, |
| + .id = ZSTD_COMPRESSION, |
| + .name = "zstd", |
| + .supported = 1 |
| +}; |
| diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h |
| new file mode 100644 |
| index 0000000..4fbef0a |
| --- /dev/null |
| +++ b/squashfs-tools/zstd_wrapper.h |
| @@ -0,0 +1,48 @@ |
| +#ifndef ZSTD_WRAPPER_H |
| +#define ZSTD_WRAPPER_H |
| +/* |
| + * Squashfs |
| + * |
| + * Copyright (c) 2017 |
| + * Phillip Lougher <phillip@squashfs.org.uk> |
| + * |
| + * This program is free software; you can redistribute it and/or |
| + * modify it under the terms of the GNU General Public License |
| + * as published by the Free Software Foundation; either version 2, |
| + * or (at your option) any later version. |
| + * |
| + * This program is distributed in the hope that it will be useful, |
| + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| + * GNU General Public License for more details. |
| + * |
| + * zstd_wrapper.h |
| + * |
| + */ |
| + |
| +#ifndef linux |
| +#define __BYTE_ORDER BYTE_ORDER |
| +#define __BIG_ENDIAN BIG_ENDIAN |
| +#define __LITTLE_ENDIAN LITTLE_ENDIAN |
| +#else |
| +#include <endian.h> |
| +#endif |
| + |
| +#if __BYTE_ORDER == __BIG_ENDIAN |
| +extern unsigned int inswap_le16(unsigned short); |
| +extern unsigned int inswap_le32(unsigned int); |
| + |
| +#define SQUASHFS_INSWAP_COMP_OPTS(s) { \ |
| + (s)->compression_level = inswap_le32((s)->compression_level); \ |
| +} |
| +#else |
| +#define SQUASHFS_INSWAP_COMP_OPTS(s) |
| +#endif |
| + |
| +/* Default compression */ |
| +#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 |
| + |
| +struct zstd_comp_opts { |
| + int compression_level; |
| +}; |
| +#endif |
| -- |
| 2.9.5 |