cli : -d and -t do not stop after a failed decompression
The problematic srcfile will be named on console/log,
but decompression/test will continue onto next file in the list.
diff --git a/programs/fileio.c b/programs/fileio.c
index e188936..ba15555 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -206,8 +206,8 @@
static int FIO_remove(const char* path)
{
#if defined(_WIN32) || defined(WIN32)
- /* windows doesn't allow remove read-only files, so try to make it
- * writable first */
+ /* windows doesn't allow remove read-only files,
+ * so try to make it writable first */
chmod(path, _S_IWRITE);
#endif
return remove(path);
@@ -983,16 +983,19 @@
/** FIO_decompressFrame() :
- @return : size of decoded frame
+ @return : size of decoded frame, or an error code
*/
+#define FIO_ERROR_ZSTD_DECODING ((unsigned long long)(-2))
unsigned long long FIO_decompressFrame(dRess_t* ress,
FILE* finput,
+ const char* srcFileName,
U64 alreadyDecoded)
{
U64 frameSize = 0;
U32 storedSkips = 0;
ZSTD_resetDStream(ress->dctx);
+ if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
/* Header loading (optional, saves one loop) */
{ size_t const toRead = 9;
@@ -1005,12 +1008,17 @@
ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 };
ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 };
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
- if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
+ if (ZSTD_isError(readSizeHint)) {
+ DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
+ srcFileName, ZSTD_getErrorName(readSizeHint));
+ return FIO_ERROR_ZSTD_DECODING;
+ }
/* Write block */
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos;
- DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
+ DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
+ srcFileName, (U32)((alreadyDecoded+frameSize)>>20) );
if (inBuff.pos > 0) {
memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
@@ -1018,14 +1026,22 @@
}
if (readSizeHint == 0) break; /* end of frame */
- if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
+ if (inBuff.size != inBuff.pos) {
+ DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n",
+ srcFileName);
+ return FIO_ERROR_ZSTD_DECODING;
+ }
/* Fill input buffer */
{ size_t const toRead = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */
if (ress->srcBufferLoaded < toRead)
- ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
- if (ress->srcBufferLoaded < toRead) EXM_THROW(39, "Read error : premature end");
- } }
+ ress->srcBufferLoaded += fread((char*)ress->srcBuffer + ress->srcBufferLoaded,
+ 1, toRead - ress->srcBufferLoaded, finput);
+ if (ress->srcBufferLoaded < toRead) {
+ DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n",
+ srcFileName);
+ return FIO_ERROR_ZSTD_DECODING;
+ } } }
FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
@@ -1221,9 +1237,14 @@
size_t const toRead = 4;
const BYTE* buf = (const BYTE*)ress.srcBuffer;
if (ress.srcBufferLoaded < toRead)
- ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
+ ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded,
+ (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
if (ress.srcBufferLoaded==0) {
- if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
+ if (readSomething==0) {
+ DISPLAY("zstd: %s: unexpected end of file \n", srcFileName);
+ fclose(srcFile);
+ return 1;
+ } /* srcFileName is empty */
break; /* no more input */
}
readSomething = 1; /* there is at least >= 4 bytes in srcFile */
@@ -1259,7 +1280,8 @@
} else {
if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
- unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
+ unsigned const result = FIO_passThrough(ress.dstFile, srcFile,
+ ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
return result;
} else {
@@ -1267,9 +1289,15 @@
fclose(srcFile);
return 1;
} }
- filesize += FIO_decompressFrame(&ress, srcFile, filesize);
+ { unsigned long long const frameSize = FIO_decompressFrame(&ress, srcFile, srcFileName, filesize);
+ if (frameSize == FIO_ERROR_ZSTD_DECODING) {
+ fclose(srcFile);
+ return 1;
+ }
+ filesize += frameSize;
+ }
}
- }
+ } /* for each frame */
/* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
@@ -1364,15 +1392,20 @@
dstFileName = (char*)malloc(dfnSize);
if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName");
}
- if (sfnSize <= suffixSize || (strcmp(suffixPtr, GZ_EXTENSION) && strcmp(suffixPtr, XZ_EXTENSION) && strcmp(suffixPtr, ZSTD_EXTENSION) && strcmp(suffixPtr, LZMA_EXTENSION) && strcmp(suffixPtr, LZ4_EXTENSION))) {
- DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s expected) -- ignored \n", srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION);
+ if (sfnSize <= suffixSize
+ || (strcmp(suffixPtr, GZ_EXTENSION)
+ && strcmp(suffixPtr, XZ_EXTENSION)
+ && strcmp(suffixPtr, ZSTD_EXTENSION)
+ && strcmp(suffixPtr, LZMA_EXTENSION)
+ && strcmp(suffixPtr, LZ4_EXTENSION)) ) {
+ DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s/%s expected) -- ignored \n",
+ srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION, LZ4_EXTENSION);
skippedFiles++;
continue;
} else {
memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
dstFileName[sfnSize-suffixSize] = '\0';
}
-
missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName);
}
free(dstFileName);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 021fd59..e695745 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -92,7 +92,7 @@
$ECHO "test : compress to named file"
rm tmpCompressed
$ZSTD tmp -o tmpCompressed
-ls tmpCompressed # must work
+test -f tmpCompressed # file must be created
$ECHO "test : -o must be followed by filename (must fail)"
$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename "
$ECHO "test : force write, correct order"
@@ -142,21 +142,21 @@
rm -f tmpro tmpro.zst
$ECHO "test : file removal"
$ZSTD -f --rm tmp
-ls tmp && die "tmp should no longer be present"
+test ! -f tmp # tmp should no longer be present
$ZSTD -f -d --rm tmp.zst
-ls tmp.zst && die "tmp.zst should no longer be present"
+test ! -f tmp.zst # tmp.zst should no longer be present
$ECHO "test : --rm on stdin"
$ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent
rm tmp
$ZSTD -f tmp && die "tmp not present : should have failed"
-ls tmp.zst && die "tmp.zst should not be created"
+test ! -f tmp.zst # tmp.zst should not be created
$ECHO "\n**** Advanced compression parameters **** "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
-ls tmp.zst && die "tmp.zst should not be created"
+test ! -f tmp.zst # tmp.zst should not be created
roundTripTest -g512K
roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
@@ -201,16 +201,17 @@
$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
+
$ECHO "\n**** symbolic link test **** "
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
$ECHO "hello world" > hello.tmp
ln -s hello.tmp world.tmp
$ZSTD world.tmp hello.tmp
-ls hello.tmp.zst || die "regular file should have been compressed!"
-ls world.tmp.zst && die "symbolic link should not have been compressed!"
+test -f hello.tmp.zst # regular file should have been compressed!
+test ! -f world.tmp.zst # symbolic link should not have been compressed!
$ZSTD world.tmp hello.tmp -f
-ls world.tmp.zst || die "symbol link should have been compressed with --force"
+test -f world.tmp.zst # symbolic link should have been compressed with --force
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
fi
@@ -225,10 +226,10 @@
$DIFF -s tmpSparse tmpOutSparse
$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse
$DIFF -s tmpSparse tmpOutNoSparse
-ls -ls tmpSparse*
+ls -ls tmpSparse* # look at file size and block size on disk
./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks)
./datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
-ls -ls tmpSparseOdd
+ls -ls tmpSparseOdd # look at file size and block size on disk
$ECHO "\n Sparse Compatibility with Console :"
$ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c
$ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat
@@ -238,7 +239,7 @@
$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed
$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
-ls -ls tmpSparse*
+ls -ls tmpSparse* # look at file size and block size on disk
$DIFF tmpSparse2M tmpSparseRegenerated
rm tmpSparse*
@@ -257,11 +258,11 @@
ls -ls tmp*
$ECHO "compress tmp* into stdout > tmpall : "
$ZSTD -c tmp1 tmp2 tmp3 > tmpall
-ls -ls tmp*
+ls -ls tmp* # check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst)
$ECHO "decompress tmpall* into stdout > tmpdec : "
cp tmpall tmpall2
$ZSTD -dc tmpall* > tmpdec
-ls -ls tmp*
+ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
$ECHO "compress multiple files including a missing one (notHere) : "
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
@@ -379,7 +380,9 @@
$ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad
$ECHO "test --rm and --test combined "
$ZSTD -t --rm tmp1.zst
-ls -ls tmp1.zst # check file is still present
+test -f tmp1.zst # check file is still present
+split -b16384 tmp1.zst tmpSplit.
+$ZSTD -t tmpSplit.* && die "bad file not detected !"
$ECHO "\n**** benchmark mode tests **** "
