changed ZSTD_BLOCKSIZE_ABSOLUTEMAX into ZSTD_BLOCKSIZE_MAX
Also :
change ZSTD_getBlockSizeMax() into ZSTD_getBlockSize()
created ZSTD_BLOCKSIZELOG_MAX
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index ff1ecd9..de0ee43 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -240,9 +240,11 @@
ZSTD_endStream() instructs to finish a frame.
It will perform a flush and write frame epilogue.
The epilogue is required for decoders to consider a frame completed.
- Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
+ ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
In which case, call again ZSTD_endStream() to complete the flush.
- @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
+ @return : 0 if frame fully completed and fully flushed,
+ or >0 if some data is still present within internal buffer
+ (value is minimum size estimation for remaining data to flush, but it could be more)
or an error code, which can be tested using ZSTD_isError().
@@ -856,7 +858,7 @@
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+ copyCCtx() and copyDCtx() can be used too
- - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
+ - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
+ For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@@ -869,7 +871,7 @@
Use ZSTD_insertBlock() for such a case.
<BR></pre>
-<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b8fc8d9..f2d40ed 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -410,7 +410,7 @@
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
{
- size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
U32 const divider = (cParams.searchLength==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = blockSize + 11*maxNbSeq;
@@ -436,8 +436,8 @@
static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
ZSTD_compressionParameters cParams2)
{
- U32 bslog1 = MIN(cParams1.windowLog, 17);
- U32 bslog2 = MIN(cParams2.windowLog, 17);
+ U32 bslog1 = MIN(cParams1.windowLog, ZSTD_BLOCKSIZELOG_MAX);
+ U32 bslog2 = MIN(cParams2.windowLog, ZSTD_BLOCKSIZELOG_MAX);
return (bslog1 == bslog2) /* same block size */
& (cParams1.hashLog == cParams2.hashLog)
& (cParams1.chainLog == cParams2.chainLog)
@@ -484,7 +484,7 @@
return ZSTD_continueCCtx(zc, params, frameContentSize);
}
- { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
+ { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
U32 const divider = (params.cParams.searchLength==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = blockSize + 11*maxNbSeq;
@@ -2807,14 +2807,18 @@
}
-size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx)
+size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
{
- return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog);
+ U32 const cLevel = cctx->compressionLevel;
+ ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ?
+ cctx->params.cParams :
+ ZSTD_getCParams(cLevel, 0, 0);
+ return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
}
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
- size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
+ size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
}
@@ -3308,7 +3312,7 @@
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
{
size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
- size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
size_t const streamingSize = inBuffSize + outBuffSize;
@@ -3319,11 +3323,11 @@
/*====== Initialization ======*/
-size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; }
size_t ZSTD_CStreamOutSize(void)
{
- return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
+ return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
}
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize)
@@ -3350,7 +3354,7 @@
unsigned long long pledgedSrcSize)
{
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
- zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
+ zcs->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
/* allocate buffers */
{ size_t const neededInBuffSize = ((size_t)1 << params.cParams.windowLog) + zcs->blockSize;
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 0c61960..03f8a92 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -117,7 +117,7 @@
ZSTD_customMem customMem;
size_t litSize;
size_t rleSize;
- BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
+ BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
@@ -173,7 +173,7 @@
void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
{
- size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
+ size_t const workSpaceSize = (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
}
@@ -483,7 +483,7 @@
litCSize = (lhc >> 22) + (istart[4] << 10);
break;
}
- if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
+ if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUF_isError((litEncType==set_repeat) ?
@@ -555,7 +555,7 @@
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
break;
}
- if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
+ if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
@@ -1288,7 +1288,7 @@
{ /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src;
- if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong);
+ if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
/* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
@@ -2120,8 +2120,8 @@
/* *** Initialization *** */
-size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
-size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
{
@@ -2185,7 +2185,7 @@
size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
{
size_t const windowSize = fHeader.windowSize;
- size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+ size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
size_t const inBuffSize = blockSize; /* no block can be larger */
size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
@@ -2281,7 +2281,7 @@
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
/* Adapt buffer sizes to frame header instructions */
- { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
zds->blockSize = blockSize;
if (zds->inBuffSize < blockSize) {
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 179e02e..943ddde 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -576,7 +576,7 @@
{
ZSTD_CCtx* ref;
ZSTD_CCtx* zc;
- void* workPlace; /* must be ZSTD_BLOCKSIZE_ABSOLUTEMAX allocated */
+ void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */
} EStats_ress_t;
#define MAXREPOFFSET 1024
@@ -585,14 +585,14 @@
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
const void* src, size_t srcSize, U32 notificationLevel)
{
- size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog);
+ size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog);
size_t cSize;
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
{ size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0);
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
}
- cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
+ cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
@@ -700,7 +700,7 @@
/* init */
esr.ref = ZSTD_createCCtx();
esr.zc = ZSTD_createCCtx();
- esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+ esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
if (!esr.ref || !esr.zc || !esr.workPlace) {
eSize = ERROR(memory_allocation);
DISPLAYLEVEL(1, "Not enough memory \n");
diff --git a/lib/zstd.h b/lib/zstd.h
index 7931bd5..5e4bc15 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1003,7 +1003,7 @@
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+ copyCCtx() and copyDCtx() can be used too
- - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
+ - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
+ For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@@ -1016,9 +1016,10 @@
Use ZSTD_insertBlock() for such a case.
*/
-#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
+#define ZSTD_BLOCKSIZELOG_MAX 17
+#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
/*===== Raw zstd block functions =====*/
-ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 6814d54..583018a 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -586,6 +586,7 @@
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
CHECK( ZSTD_compressBegin(cctx, 5) );
+ CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
