implemented ZSTD_refPrefix()
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 69b6fbf..f5a1394 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -348,7 +348,7 @@
<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
</b><p> `src` should point to the start of a ZSTD encoded frame or skippable frame
`srcSize` must be at least as large as the frame
- @return : the compressed size of the frame pointed to by `src`,FUZ_rand(&lseed) & 1)
+ @return : the compressed size of the frame pointed to by `src`,
suitable to pass to `ZSTD_decompress` or similar,
or an error code if given invalid input.
</p></pre><BR>
@@ -806,20 +806,13 @@
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "do not change strategy". */
-#if 0
- ZSTD_p_windowSize, </b>/* Maximum allowed back-reference distance.<b>
- * Can be set to a more precise value than windowLog.
- * Will be transparently reduced to closest possible inferior value
- * (see Zstandard compression format) */
- </b>/* Not ready yet ! */<b>
-#endif
</b>/* frame parameters */<b>
ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1) */<b>
ZSTD_p_checksumFlag, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
ZSTD_p_dictIDFlag, </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b>
- </b>/* dictionary parameters (must be set before loading) */<b>
+ </b>/* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */<b>
ZSTD_p_dictMode=300, </b>/* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.<b>
* default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
ZSTD_p_refDictContent, </b>/* Dictionary content will be referenced, instead of copied (default:0==byCopy).<b>
@@ -876,7 +869,7 @@
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-</b><p> Ref a prepared dictionary, to be used for all next compression jobs.
+</b><p> Reference a prepared dictionary, to be used for all next compression jobs.
Note that compression parameters are enforced from within CDict,
and supercede any compression parameter previously set within CCtx.
The dictionary will remain valid for future compression jobs using same CCtx.
@@ -888,16 +881,18 @@
</p></pre><BR>
-<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); </b>/* Not ready yet ! <===================================== */<b>
-</b><p> Reference a prefix (raw-content dictionary) for next compression job.
- Decompression will have to use same prefix.
- Prefix is only used once. Tables are discarded at end of compression job.
- If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
+<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
+</b><p> Reference a prefix (single-usage dictionary) for next compression job.
+ Decompression need same prefix to properly regenerate data.
+ Prefix is **only used once**. Tables are discarded at end of compression job.
+ Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
+ If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
- Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
+ Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
Note 1 : Prefix buffer is referenced. It must outlive compression job.
Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
- It's a CPU-heavy operation, with non-negligible impact on latency.
+ It's a CPU-heavy operation, with non-negligible impact on latency.
+ Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter()
</p></pre><BR>
<pre><b>typedef enum {
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b6b92f3..93400c5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -88,8 +88,6 @@
U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; /* index of end of dictionary */
U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
- ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
- U32 dictContentByRef;
ZSTD_compressionStage_e stage;
U32 rep[ZSTD_REP_NUM];
U32 repToConfirm[ZSTD_REP_NUM];
@@ -119,8 +117,6 @@
unsigned* entropyScratchSpace;
/* streaming */
- ZSTD_CDict* cdictLocal;
- const ZSTD_CDict* cdict;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
@@ -133,6 +129,14 @@
ZSTD_cStreamStage streamStage;
U32 frameEnded;
+ /* Dictionary */
+ ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
+ U32 dictContentByRef;
+ ZSTD_CDict* cdictLocal;
+ const ZSTD_CDict* cdict;
+ const void* prefix;
+ size_t prefixSize;
+
/* Multi-threading */
U32 nbThreads;
ZSTDMT_CCtx* mtctx;
@@ -427,19 +431,22 @@
return 0;
}
-/* Not ready yet ! */
-size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
-{
- (void)cctx; (void)prefix; (void)prefixSize; /* to be done later */
- if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
- return ERROR(compressionParameter_unsupported);
-}
-
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
cctx->cdict = cdict;
- return ERROR(compressionParameter_unsupported);
+ cctx->prefix = NULL; /* exclusive */
+ cctx->prefixSize = 0;
+ return 0;
+}
+
+size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
+{
+ if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+ cctx->cdict = NULL; /* prefix discards any prior cdict */
+ cctx->prefix = prefix;
+ cctx->prefixSize = prefixSize;
+ return 0;
}
static void ZSTD_startNewCompression(ZSTD_CCtx* cctx)
@@ -3622,14 +3629,17 @@
}
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
- ZSTD_parameters params,
- unsigned long long pledgedSrcSize)
+ const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
DEBUGLOG(5, "ZSTD_resetCStream_internal");
+ /* params are supposed to be fully validated at this point */
+ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
CHECK_F( ZSTD_compressBegin_internal(zcs,
- NULL, 0, ZSTD_dm_auto,
- zcs->cdict,
+ dict, dictSize, ZSTD_dm_auto, /* <========= Todo : make dictMode controllable ! */
+ cdict,
params, pledgedSrcSize,
ZSTDb_buffered) );
@@ -3650,10 +3660,11 @@
if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
}
- return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
+ return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
}
/*! ZSTD_initCStream_internal() :
+ * Note : not static, but hidden (not exposed). Used by zstdmt_compress.c
* Assumption 1 : params are valid
* Assumption 2 : either dict, or cdict, is defined, not both */
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
@@ -3672,7 +3683,7 @@
}
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
- 0 /* byReference */, ZSTD_dm_auto,
+ 0 /* byReference */, ZSTD_dm_auto, /* <======== Todo : make dictMode controllable */
params.cParams, zcs->customMem);
zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@@ -3688,7 +3699,7 @@
zcs->requestedParams = params;
zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
- return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
+ return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
}
/* ZSTD_initCStream_usingCDict_advanced() :
@@ -3925,7 +3936,10 @@
} else
#endif
{
- CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->pledgedSrcSizePlusOne-1) );
+ const void* const prefix = cctx->prefix;
+ size_t const prefixSize = cctx->prefixSize;
+ cctx->prefix = NULL; cctx->prefixSize = 0; /* single usage */
+ CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
} }
#ifdef ZSTD_MULTITHREAD
diff --git a/lib/zstd.h b/lib/zstd.h
index 2b90187..700a22f 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -947,20 +947,13 @@
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "do not change strategy". */
-#if 0
- ZSTD_p_windowSize, /* Maximum allowed back-reference distance.
- * Can be set to a more precise value than windowLog.
- * Will be transparently reduced to closest possible inferior value
- * (see Zstandard compression format) */
- /* Not ready yet ! */
-#endif
/* frame parameters */
ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
- /* dictionary parameters (must be set before loading) */
+ /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
* default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
ZSTD_p_refDictContent, /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
@@ -1018,7 +1011,7 @@
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_refCDict() :
- * Ref a prepared dictionary, to be used for all next compression jobs.
+ * Reference a prepared dictionary, to be used for all next compression jobs.
* Note that compression parameters are enforced from within CDict,
* and supercede any compression parameter previously set within CCtx.
* The dictionary will remain valid for future compression jobs using same CCtx.
@@ -1031,16 +1024,18 @@
ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
/*! ZSTD_CCtx_refPrefix() :
- * Reference a prefix (raw-content dictionary) for next compression job.
- * Decompression will have to use same prefix.
- * Prefix is only used once. Tables are discarded at end of compression job.
- * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
+ * Reference a prefix (single-usage dictionary) for next compression job.
+ * Decompression need same prefix to properly regenerate data.
+ * Prefix is **only used once**. Tables are discarded at end of compression job.
+ * Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
+ * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
+ * Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
* Note 1 : Prefix buffer is referenced. It must outlive compression job.
* Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
- * It's a CPU-heavy operation, with non-negligible impact on latency. */
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); /* Not ready yet ! <===================================== */
+ * It's a CPU-heavy operation, with non-negligible impact on latency.
+ * Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index ca25e89..0f20d32 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1266,11 +1266,11 @@
}
/* compression init */
+ CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
&& oldTestLog /* at least one test happened */ && resetAllowed) {
maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
- CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );
{ int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, compressionLevel) );
}
@@ -1294,7 +1294,6 @@
ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
/* mess with compression parameters */
- CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict, to allow new compression parameters */
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
@@ -1312,7 +1311,8 @@
/* unconditionally set, to be sync with decoder */
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
- CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
+ if (FUZ_rand(&lseed) & 1) { CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); }
+ else { CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); }
if (dict && dictSize) {
/* test that compression parameters are correctly rejected after setting a dictionary */
