[med-svn] [libzstd] 01/04: Imported Upstream version 0.4.5
Kevin Murray
daube-guest at moszumanska.debian.org
Tue Dec 22 01:51:16 UTC 2015
This is an automated email from the git hooks/post-receive script.
daube-guest pushed a commit to branch master
in repository libzstd.
commit 4b8b26a95669a7325d5a356e31141bbb21112c1e
Author: Kevin Murray <spam at kdmurray.id.au>
Date: Tue Dec 22 12:43:51 2015 +1100
Imported Upstream version 0.4.5
---
.gitignore | 18 +
.travis.yml | 2 +
Makefile | 11 +-
NEWS | 22 +-
images/CSpeed.png | Bin 37278 -> 36039 bytes
lib/Makefile | 24 +-
lib/zstd.c | 1462 --------------------
lib/zstd.h | 40 +-
lib/zstd_buffered.c | 20 +-
lib/zstd_buffered.h | 56 +-
lib/zstd_buffered_static.h | 2 +-
lib/zstd_compress.c | 177 ++-
lib/zstd_decompress.c | 191 +--
lib/zstd_static.h | 139 +-
programs/Makefile | 108 +-
programs/bench.c | 404 +++---
programs/bench.h | 3 +-
programs/datagen.c | 1 +
programs/datagen.h | 2 +-
programs/fileio.c | 465 +++++--
programs/fileio.h | 26 +-
programs/fullbench.c | 5 +-
programs/fuzzer.c | 126 +-
programs/paramgrill.c | 4 +
programs/playTests.sh | 104 ++
programs/zbufftest.c | 17 +-
programs/zstd.1 | 34 +-
programs/zstdcli.c | 143 +-
visual/{2012 => 2013}/fullbench/fullbench.vcxproj | 14 +-
.../fullbench/fullbench.vcxproj.filters | 0
visual/{2012 => 2013}/fuzzer/fuzzer.vcxproj | 14 +-
.../{2012 => 2013}/fuzzer/fuzzer.vcxproj.filters | 0
visual/{2012 => 2013}/zstd.sln | 14 +-
visual/{2012 => 2013}/zstd/zstd.vcxproj | 12 +-
visual/{2012 => 2013}/zstd/zstd.vcxproj.filters | 6 +
visual/2013/zstdlib/resource.h | Bin 0 -> 812 bytes
visual/2013/zstdlib/zstdlib.rc | Bin 0 -> 4610 bytes
.../zstd.vcxproj => 2013/zstdlib/zstdlib.vcxproj} | 92 +-
visual/2013/zstdlib/zstdlib.vcxproj.filters | 80 ++
39 files changed, 1617 insertions(+), 2221 deletions(-)
diff --git a/.gitignore b/.gitignore
index c4c9282..072fe39 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,21 @@
# Visual solution files
*.suo
*.user
+
+# Build results
+[Dd]ebug/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Bb]in/
+[Oo]bj/
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
index 83a6f23..2d5a73a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,8 +16,10 @@ env:
- ZSTD_TRAVIS_CI_ENV=armtest
- ZSTD_TRAVIS_CI_ENV=test
- ZSTD_TRAVIS_CI_ENV="-C programs test32"
+ - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy"
- ZSTD_TRAVIS_CI_ENV=usan
- ZSTD_TRAVIS_CI_ENV=asan
+ - ZSTD_TRAVIS_CI_ENV=asan32
- ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest"
matrix:
diff --git a/Makefile b/Makefile
index 4ec4a79..a741034 100644
--- a/Makefile
+++ b/Makefile
@@ -32,7 +32,7 @@
# ################################################################
# Version number
-export VERSION := 0.4.0
+export VERSION := 0.4.5
PRGDIR = programs
ZSTDDIR = lib
@@ -74,7 +74,7 @@ uninstall:
$(MAKE) -C $(PRGDIR) $@
travis-install:
- sudo $(MAKE) install
+ $(MAKE) install PREFIX=~/install_test_dir
test:
$(MAKE) -C $(PRGDIR) $@
@@ -87,8 +87,8 @@ gpptest: clean
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
armtest: clean
- $(MAKE) -C $(ZSTDDIR) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
- $(MAKE) -C $(PRGDIR) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
+ $(MAKE) -C $(ZSTDDIR) all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
+ $(MAKE) -C $(PRGDIR) CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror -static"
usan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@@ -96,6 +96,9 @@ usan: clean
asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
+asan32: clean
+ $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
+
uasan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
diff --git a/NEWS b/NEWS
index 75c9b87..ee9a458 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,25 @@
+v0.4.5
+new : -m/--multiple : compress/decompress multiple files
+
+v0.4.4
+Fixed : high compression modes for Windows 32 bits
+new : external dictionary API extended to buffered mode and accessible through command line
+new : windows DLL project, thanks to Christophe Chevalier
+
+v0.4.3 :
+new : external dictionary API
+new : zstd-frugal
+
+v0.4.2 :
+Generic minor improvements for small blocks
+Fixed : big-endian compatibility, by Peter Harris (#85)
+
+v0.4.1
+Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
+removed `zstd.c`
+
v0.4.0
-Command line utility is now compatible with high compression levels
+Command line utility compatible with high compression levels
Removed zstdhc => merged into zstd
Added : ZBUFF API (see zstd_buffered.h)
Rolling buffer support
diff --git a/images/CSpeed.png b/images/CSpeed.png
index 0289e2d..fe54752 100644
Binary files a/images/CSpeed.png and b/images/CSpeed.png differ
diff --git a/lib/Makefile b/lib/Makefile
index 3e34d89..c621f63 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -40,7 +40,7 @@ VERSION?= $(LIBVER)
DESTDIR?=
PREFIX ?= /usr/local
-CPPFLAGS= -I. -I./legacy -DZSTD_LEGACY_SUPPORT=1
+CPPFLAGS= -I.
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
@@ -48,6 +48,16 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
+ZSTD_FILES := zstd_compress.c zstd_decompress.c fse.c huff0.c
+ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c
+
+ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
+CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
+else
+ZSTD_FILES+= $(ZSTD_LEGACY)
+CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
+endif
+
# OS X linker doesn't support -soname, and use different extension
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
@@ -55,9 +65,9 @@ ifeq ($(shell uname), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
- SONAME_FLAGS = -install_name $(PREFIX)/lib/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+ SONAME_FLAGS = -install_name $(PREFIX)/lib/$@.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
else
- SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
+ SONAME_FLAGS = -Wl,-soname=$@.$(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT = so
SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
@@ -70,11 +80,10 @@ default: clean libzstd
all: clean libzstd
-libzstd: zstd_compress.c zstd_decompress.c huff0.c fse.c \
- legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c
+libzstd: $(ZSTD_FILES)
@echo compiling static library
@$(CC) $(FLAGS) -c $^
- @$(AR) rcs libzstd.a *.o
+ @$(AR) rcs $@.a *.o
@echo compiling dynamic library $(LIBVER)
@$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
@echo creating versioned links
@@ -82,10 +91,9 @@ libzstd: zstd_compress.c zstd_decompress.c huff0.c fse.c \
@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT)
clean:
- @rm -f core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
+ @rm -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
@echo Cleaning library completed
-
#------------------------------------------------------------------------
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
diff --git a/lib/zstd.c b/lib/zstd.c
deleted file mode 100644
index a76d111..0000000
--- a/lib/zstd.c
+++ /dev/null
@@ -1,1462 +0,0 @@
-/*
- zstd - standard compression library
- Copyright (C) 2014-2015, Yann Collet.
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - zstd source repository : https://github.com/Cyan4973/zstd
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-
-/* ***************************************************************
-* Tuning parameters
-*****************************************************************/
-/*!
-* MEMORY_USAGE :
-* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
-* Increasing memory usage improves compression ratio
-* Reduced memory usage can improve speed, due to cache effect
-*/
-#define ZSTD_MEMORY_USAGE 16
-
-/*!
- * HEAPMODE :
- * Select how default compression functions will allocate memory for their hash table,
- * in memory stack (0, fastest), or in memory heap (1, requires malloc())
- * Note that compression context is fairly large, as a consequence heap memory is recommended.
- */
-#ifndef ZSTD_HEAPMODE
-# define ZSTD_HEAPMODE 1
-#endif /* ZSTD_HEAPMODE */
-
-/*!
-* LEGACY_SUPPORT :
-* decompressor can decode older formats (starting from Zstd 0.1+)
-*/
-#ifndef ZSTD_LEGACY_SUPPORT
-# define ZSTD_LEGACY_SUPPORT 1
-#endif
-
-
-/* *******************************************************
-* Includes
-*********************************************************/
-#include <stdlib.h> /* calloc */
-#include <string.h> /* memcpy, memmove */
-#include <stdio.h> /* debug : printf */
-#include "mem.h" /* low level memory routines */
-#include "zstd_static.h"
-#include "zstd_internal.h"
-#include "fse_static.h"
-#include "huff0.h"
-
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-# include "zstd_legacy.h"
-#endif
-
-
-/* *******************************************************
-* Compiler specifics
-*********************************************************/
-#ifdef __AVX2__
-# include <immintrin.h> /* AVX2 intrinsics */
-#endif
-
-#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
-# include <intrin.h> /* For Visual 2005 */
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-# pragma warning(disable : 4324) /* disable: C4324: padded structure */
-#else
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
-#endif
-
-
-/* *******************************************************
-* Constants
-*********************************************************/
-#define HASH_LOG (ZSTD_MEMORY_USAGE - 2)
-#define HASH_TABLESIZE (1 << HASH_LOG)
-#define HASH_MASK (HASH_TABLESIZE - 1)
-
-#define KNUTH 2654435761
-
-#define BIT7 128
-#define BIT6 64
-#define BIT5 32
-#define BIT4 16
-#define BIT1 2
-#define BIT0 1
-
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
-
-#define BLOCKSIZE (128 KB) /* define, for static allocation */
-#define IS_RAW BIT0
-#define IS_RLE BIT1
-
-static const U32 g_maxDistance = 4 * BLOCKSIZE;
-static const U32 g_maxLimit = 1 GB;
-
-#define WORKPLACESIZE (BLOCKSIZE*3)
-#define MINMATCH 4
-#define LitFSELog 11
-#define MLFSELog 10
-#define LLFSELog 10
-#define OffFSELog 9
-#define MAX(a,b) ((a)<(b)?(b):(a))
-#define MaxSeq MAX(MaxLL, MaxML)
-
-#define LITERAL_NOENTROPY 63
-#define COMMAND_NOENTROPY 7 /* to remove */
-
-static const size_t ZSTD_blockHeaderSize = 3;
-static const size_t ZSTD_frameHeaderSize = 4;
-
-
-/* *******************************************************
-* Memory operations
-**********************************************************/
-static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
-
-
-/* **************************************
-* Local structures
-****************************************/
-void ZSTD_resetSeqStore(seqStore_t* ssPtr)
-{
- ssPtr->offset = ssPtr->offsetStart;
- ssPtr->lit = ssPtr->litStart;
- ssPtr->litLength = ssPtr->litLengthStart;
- ssPtr->matchLength = ssPtr->matchLengthStart;
- ssPtr->dumps = ssPtr->dumpsStart;
-}
-
-struct ZSTD_CCtx_s
-{
- const BYTE* base;
- U32 current;
- U32 nextUpdate;
- seqStore_t seqStore;
-#ifdef __AVX2__
- __m256i hashTable[HASH_TABLESIZE>>3];
-#else
- U32 hashTable[HASH_TABLESIZE];
-#endif
- BYTE buffer[WORKPLACESIZE];
-};
-
-
-void ZSTD_resetCCtx(ZSTD_CCtx* ctx)
-{
- ctx->base = NULL;
- ctx->seqStore.buffer = ctx->buffer;
- ctx->seqStore.offsetStart = (U32*) (ctx->seqStore.buffer);
- ctx->seqStore.offCodeStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2));
- ctx->seqStore.litStart = ctx->seqStore.offCodeStart + (BLOCKSIZE>>2);
- ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE;
- ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2);
- ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2);
- memset(ctx->hashTable, 0, sizeof(ctx->hashTable));
-}
-
-ZSTD_CCtx* ZSTD_createCCtx(void)
-{
- ZSTD_CCtx* ctx = (ZSTD_CCtx*) malloc( sizeof(ZSTD_CCtx) );
- if (ctx==NULL) return NULL;
- ZSTD_resetCCtx(ctx);
- return ctx;
-}
-
-size_t ZSTD_freeCCtx(ZSTD_CCtx* ctx)
-{
- free(ctx);
- return 0;
-}
-
-
-/* *************************************
-* Error Management
-***************************************/
-/*! ZSTD_isError
-* tells if a return value is an error code */
-unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
-
-/*! ZSTD_getErrorName
-* provides error code string (useful for debugging) */
-const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
-
-
-/* *************************************
-* Tool functions
-***************************************/
-unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
-
-
-/* *******************************************************
-* Compression
-*********************************************************/
-size_t ZSTD_compressBound(size_t srcSize) /* maximum compressed size */
-{
- return FSE_compressBound(srcSize) + 12;
-}
-
-
-size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- BYTE* const ostart = (BYTE* const)dst;
-
- if (srcSize + ZSTD_blockHeaderSize > maxDstSize) return ERROR(dstSize_tooSmall);
- memcpy(ostart + ZSTD_blockHeaderSize, src, srcSize);
-
- /* Build header */
- ostart[0] = (BYTE)(srcSize>>16);
- ostart[1] = (BYTE)(srcSize>>8);
- ostart[2] = (BYTE) srcSize;
- ostart[0] += (BYTE)(bt_raw<<6); /* is a raw (uncompressed) block */
-
- return ZSTD_blockHeaderSize+srcSize;
-}
-
-
-static size_t ZSTD_compressRawLiteralsBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- BYTE* const ostart = (BYTE* const)dst;
-
- if (srcSize + 3 > maxDstSize) return ERROR(dstSize_tooSmall);
-
- MEM_writeLE32(dst, ((U32)srcSize << 2) | IS_RAW);
- memcpy(ostart + 3, src, srcSize);
- return srcSize + 3;
-}
-
-static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- BYTE* const ostart = (BYTE* const)dst;
-
- (void)maxDstSize;
- MEM_writeLE32(dst, ((U32)srcSize << 2) | IS_RLE); /* note : maxDstSize > litHeaderSize > 4 */
- ostart[3] = *(const BYTE*)src;
- return 4;
-}
-
-size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 1; }
-
-static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
- const void* src, size_t srcSize)
-{
- const size_t minGain = ZSTD_minGain(srcSize);
- BYTE* const ostart = (BYTE*)dst;
- size_t hsize;
- static const size_t litHeaderSize = 5;
-
- if (maxDstSize < litHeaderSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */
-
- hsize = HUF_compress(ostart+litHeaderSize, maxDstSize-litHeaderSize, src, srcSize);
-
- if ((hsize==0) || (hsize >= srcSize - minGain)) return ZSTD_compressRawLiteralsBlock(dst, maxDstSize, src, srcSize);
- if (hsize==1) return ZSTD_compressRleLiteralsBlock(dst, maxDstSize, src, srcSize);
-
- /* Build header */
- {
- ostart[0] = (BYTE)(srcSize << 2); /* is a block, is compressed */
- ostart[1] = (BYTE)(srcSize >> 6);
- ostart[2] = (BYTE)(srcSize >>14);
- ostart[2] += (BYTE)(hsize << 5);
- ostart[3] = (BYTE)(hsize >> 3);
- ostart[4] = (BYTE)(hsize >>11);
- }
-
- return hsize+litHeaderSize;
-}
-
-
-size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
- const seqStore_t* seqStorePtr,
- size_t srcSize)
-{
- U32 count[MaxSeq+1];
- S16 norm[MaxSeq+1];
- size_t mostFrequent;
- U32 max = 255;
- U32 tableLog = 11;
- U32 CTable_LitLength [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL )];
- U32 CTable_OffsetBits [FSE_CTABLE_SIZE_U32(OffFSELog,MaxOff)];
- U32 CTable_MatchLength[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML )];
- U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
- const BYTE* const op_lit_start = seqStorePtr->litStart;
- const BYTE* const llTable = seqStorePtr->litLengthStart;
- const BYTE* const llPtr = seqStorePtr->litLength;
- const BYTE* const mlTable = seqStorePtr->matchLengthStart;
- const U32* const offsetTable = seqStorePtr->offsetStart;
- BYTE* const offCodeTable = seqStorePtr->offCodeStart;
- BYTE* op = dst;
- BYTE* const oend = dst + maxDstSize;
- const size_t nbSeq = llPtr - llTable;
- const size_t minGain = ZSTD_minGain(srcSize);
- const size_t maxCSize = srcSize - minGain;
- BYTE* seqHead;
-
-
- /* Compress literals */
- {
- size_t cSize;
- size_t litSize = seqStorePtr->lit - op_lit_start;
-
- if (litSize <= LITERAL_NOENTROPY)
- cSize = ZSTD_compressRawLiteralsBlock(op, maxDstSize, op_lit_start, litSize);
- else
- cSize = ZSTD_compressLiterals(op, maxDstSize, op_lit_start, litSize);
- if (ZSTD_isError(cSize)) return cSize;
- op += cSize;
- }
-
- /* Sequences Header */
- if ((oend-op) < MIN_SEQUENCES_SIZE)
- return ERROR(dstSize_tooSmall);
- MEM_writeLE16(op, (U16)nbSeq); op+=2;
- seqHead = op;
-
- /* dumps : contains too large lengths */
- {
- size_t dumpsLength = seqStorePtr->dumps - seqStorePtr->dumpsStart;
- if (dumpsLength < 512)
- {
- op[0] = (BYTE)(dumpsLength >> 8);
- op[1] = (BYTE)(dumpsLength);
- op += 2;
- }
- else
- {
- op[0] = 2;
- op[1] = (BYTE)(dumpsLength>>8);
- op[2] = (BYTE)(dumpsLength);
- op += 3;
- }
- if ((size_t)(oend-op) < dumpsLength+6) return ERROR(dstSize_tooSmall);
- memcpy(op, seqStorePtr->dumpsStart, dumpsLength);
- op += dumpsLength;
- }
-
- /* CTable for Literal Lengths */
- max = MaxLL;
- mostFrequent = FSE_countFast(count, &max, seqStorePtr->litLengthStart, nbSeq);
- if ((mostFrequent == nbSeq) && (nbSeq > 2))
- {
- *op++ = *(seqStorePtr->litLengthStart);
- FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
- LLtype = bt_rle;
- }
- else if ((nbSeq < 64) || (mostFrequent < (nbSeq >> (LLbits-1))))
- {
- FSE_buildCTable_raw(CTable_LitLength, LLbits);
- LLtype = bt_raw;
- }
- else
- {
- size_t NCountSize;
- tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
- FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
- NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
- if (FSE_isError(NCountSize)) return ERROR(GENERIC);
- op += NCountSize;
- FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
- LLtype = bt_compressed;
- }
-
- /* CTable for Offsets codes */
- {
- /* create Offset codes */
- size_t i;
- max = MaxOff;
- for (i=0; i<nbSeq; i++)
- {
- offCodeTable[i] = (BYTE)ZSTD_highbit(offsetTable[i]) + 1;
- if (offsetTable[i]==0) offCodeTable[i]=0;
- }
- mostFrequent = FSE_countFast(count, &max, offCodeTable, nbSeq);
- }
- if ((mostFrequent == nbSeq) && (nbSeq > 2))
- {
- *op++ = *offCodeTable;
- FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
- Offtype = bt_rle;
- }
- else if ((nbSeq < 64) || (mostFrequent < (nbSeq >> (Offbits-1))))
- {
- FSE_buildCTable_raw(CTable_OffsetBits, Offbits);
- Offtype = bt_raw;
- }
- else
- {
- size_t NCountSize;
- tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
- FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
- NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
- if (FSE_isError(NCountSize)) return ERROR(GENERIC);
- op += NCountSize;
- FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
- Offtype = bt_compressed;
- }
-
- /* CTable for MatchLengths */
- max = MaxML;
- mostFrequent = FSE_countFast(count, &max, seqStorePtr->matchLengthStart, nbSeq);
- if ((mostFrequent == nbSeq) && (nbSeq > 2))
- {
- *op++ = *seqStorePtr->matchLengthStart;
- FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
- MLtype = bt_rle;
- }
- else if ((nbSeq < 64) || (mostFrequent < (nbSeq >> (MLbits-1))))
- {
- FSE_buildCTable_raw(CTable_MatchLength, MLbits);
- MLtype = bt_raw;
- }
- else
- {
- size_t NCountSize;
- tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
- FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
- NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
- if (FSE_isError(NCountSize)) return ERROR(GENERIC);
- op += NCountSize;
- FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
- MLtype = bt_compressed;
- }
-
- seqHead[0] += (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
-
- /* Encoding Sequences */
- {
- size_t streamSize, errorCode;
- BIT_CStream_t blockStream;
- FSE_CState_t stateMatchLength;
- FSE_CState_t stateOffsetBits;
- FSE_CState_t stateLitLength;
- int i;
-
- errorCode = BIT_initCStream(&blockStream, op, oend-op);
- if (ERR_isError(errorCode)) return ERROR(dstSize_tooSmall); /* not enough space remaining */
- FSE_initCState(&stateMatchLength, CTable_MatchLength);
- FSE_initCState(&stateOffsetBits, CTable_OffsetBits);
- FSE_initCState(&stateLitLength, CTable_LitLength);
-
- for (i=(int)nbSeq-1; i>=0; i--)
- {
- BYTE matchLength = mlTable[i];
- U32 offset = offsetTable[i];
- BYTE offCode = offCodeTable[i]; /* 32b*/ /* 64b*/
- U32 nbBits = (offCode-1) * (!!offCode);
- BYTE litLength = llTable[i]; /* (7)*/ /* (7)*/
- FSE_encodeSymbol(&blockStream, &stateMatchLength, matchLength); /* 17 */ /* 17 */
- if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
- BIT_addBits(&blockStream, offset, nbBits); /* 32 */ /* 42 */
- if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
- FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 16 */ /* 51 */
- FSE_encodeSymbol(&blockStream, &stateLitLength, litLength); /* 26 */ /* 61 */
- BIT_flushBits(&blockStream); /* 7 */ /* 7 */
- }
-
- FSE_flushCState(&blockStream, &stateMatchLength);
- FSE_flushCState(&blockStream, &stateOffsetBits);
- FSE_flushCState(&blockStream, &stateLitLength);
-
- streamSize = BIT_closeCStream(&blockStream);
- if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */
- op += streamSize;
- }
-
- /* check compressibility */
- if ((size_t)(op-dst) >= maxCSize) return 0;
-
- return op - dst;
-}
-
-
-//static const U32 hashMask = (1<<HASH_LOG)-1;
-
-//static U32 ZSTD_hashPtr(const void* p) { return (U32) _bextr_u64(*(U64*)p * prime7bytes, (56-HASH_LOG), HASH_LOG); }
-//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime7bytes) << 8 >> (64-HASH_LOG)); }
-//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime7bytes) >> (56-HASH_LOG)) & ((1<<HASH_LOG)-1); }
-//static U32 ZSTD_hashPtr(const void* p) { return ( ((*(U64*)p & 0xFFFFFFFFFFFFFF) * prime7bytes) >> (64-HASH_LOG)); }
-
-//static const U64 prime8bytes = 14923729446516375013ULL;
-//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime8bytes) >> (64-HASH_LOG)); }
-
-static const U64 prime7bytes = 58295818150454627ULL;
-static U32 ZSTD_hashPtr(const void* p) { return ( (MEM_read64(p) * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; }
-
-//static const U64 prime6bytes = 227718039650203ULL;
-//static U32 ZSTD_hashPtr(const void* p) { return ( (MEM_read64(p) * prime6bytes) >> (48-HASH_LOG)) & HASH_MASK; }
-
-//static const U64 prime5bytes = 889523592379ULL;
-//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime5bytes) >> (40-HASH_LOG)) & HASH_MASK; }
-
-//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U32*)p * KNUTH) >> (32-HASH_LOG)); }
-
-static const BYTE* ZSTD_updateMatch(U32* table, const BYTE* p, const BYTE* start)
-{
- U32 h = ZSTD_hashPtr(p);
- const BYTE* r;
- r = table[h] + start;
- table[h] = (U32)(p-start);
- return r;
-}
-
-static int ZSTD_checkMatch(const BYTE* match, const BYTE* ip)
-{
- return MEM_read32(match) == MEM_read32(ip);
-}
-
-static void ZSTD_addPtr(U32* table, const BYTE* p, const BYTE* start) { table[ZSTD_hashPtr(p)] = (U32)(p-start); }
-
-
-static size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- U32* HashTable = (U32*)(ctx->hashTable);
- seqStore_t* seqStorePtr = &(ctx->seqStore);
- const BYTE* const base = ctx->base;
-
- const BYTE* const istart = (const BYTE*)src;
- const BYTE* ip = istart + 1;
- const BYTE* anchor = istart;
- const BYTE* const iend = istart + srcSize;
- const BYTE* const ilimit = iend - 8;
-
- size_t offset_2=4, offset_1=4;
-
-
- /* init */
- if (ip-base < 4)
- {
- ZSTD_addPtr(HashTable, ip+0, base);
- ZSTD_addPtr(HashTable, ip+1, base);
- ZSTD_addPtr(HashTable, ip+2, base);
- ZSTD_addPtr(HashTable, ip+3, base);
- ip += 4;
- }
- ZSTD_resetSeqStore(seqStorePtr);
-
- /* Main Search Loop */
- while (ip < ilimit) /* < instead of <=, because unconditionnal ZSTD_addPtr(ip+1) */
- {
- const BYTE* match = ZSTD_updateMatch(HashTable, ip, base);
-
- if (ZSTD_checkMatch(ip-offset_2,ip)) match = ip-offset_2;
- if (!ZSTD_checkMatch(match,ip)) { ip += ((ip-anchor) >> g_searchStrength) + 1; offset_2 = offset_1; continue; }
- while ((ip>anchor) && (match>base) && (ip[-1] == match[-1])) { ip--; match--; } /* catch up */
-
- {
- size_t litLength = ip-anchor;
- size_t matchLength = ZSTD_count(ip+MINMATCH, match+MINMATCH, iend);
- size_t offsetCode = ip-match;
- if (offsetCode == offset_2) offsetCode = 0;
- offset_2 = offset_1;
- offset_1 = ip-match;
- ZSTD_storeSeq(seqStorePtr, litLength, anchor, offsetCode, matchLength);
-
- /* Fill Table */
- ZSTD_addPtr(HashTable, ip+1, base);
- ip += matchLength + MINMATCH;
- anchor = ip;
- if (ip < ilimit) /* same test as loop, for speed */
- ZSTD_addPtr(HashTable, ip-2, base);
- }
- }
-
- /* Last Literals */
- {
- size_t lastLLSize = iend - anchor;
- memcpy(seqStorePtr->lit, anchor, lastLLSize);
- seqStorePtr->lit += lastLLSize;
- }
-
- /* Finale compression stage */
- return ZSTD_compressSequences((BYTE*)dst, maxDstSize,
- seqStorePtr, srcSize);
-}
-
-
-size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
-{
- /* Sanity check */
- if (maxDstSize < ZSTD_frameHeaderSize) return ERROR(dstSize_tooSmall);
-
- /* Init */
- ZSTD_resetCCtx(ctx);
-
- /* Write Header */
- MEM_writeLE32(dst, ZSTD_magicNumber);
-
- return ZSTD_frameHeaderSize;
-}
-
-
-static void ZSTD_scaleDownCtx(ZSTD_CCtx* ctx, const U32 limit)
-{
- int i;
-
-#if defined(__AVX2__)
- /* AVX2 version */
- __m256i* h = ctx->hashTable;
- const __m256i limit8 = _mm256_set1_epi32(limit);
- for (i=0; i<(HASH_TABLESIZE>>3); i++)
- {
- __m256i src =_mm256_loadu_si256((const __m256i*)(h+i));
- const __m256i dec = _mm256_min_epu32(src, limit8);
- src = _mm256_sub_epi32(src, dec);
- _mm256_storeu_si256((__m256i*)(h+i), src);
- }
-#else
- /* this should be auto-vectorized by compiler */
- U32* h = ctx->hashTable;
- for (i=0; i<HASH_TABLESIZE; ++i)
- {
- U32 dec;
- if (h[i] > limit) dec = limit; else dec = h[i];
- h[i] -= dec;
- }
-#endif
-}
-
-
-static void ZSTD_limitCtx(ZSTD_CCtx* ctx, const U32 limit)
-{
- int i;
-
- if (limit > g_maxLimit)
- {
- ZSTD_scaleDownCtx(ctx, limit);
- ctx->base += limit;
- ctx->current -= limit;
- ctx->nextUpdate -= limit;
- return;
- }
-
-#if defined(__AVX2__)
- /* AVX2 version */
- {
- __m256i* h = ctx->hashTable;
- const __m256i limit8 = _mm256_set1_epi32(limit);
- //printf("Address h : %0X\n", (U32)h); // address test
- for (i=0; i<(HASH_TABLESIZE>>3); i++)
- {
- __m256i src =_mm256_loadu_si256((const __m256i*)(h+i)); // Unfortunately, clang doesn't guarantee 32-bytes alignment
- src = _mm256_max_epu32(src, limit8);
- _mm256_storeu_si256((__m256i*)(h+i), src);
- }
- }
-#else
- /* this should be auto-vectorized by compiler */
- {
- U32* h = (U32*)(ctx->hashTable);
- for (i=0; i<HASH_TABLESIZE; ++i)
- {
- if (h[i] < limit) h[i] = limit;
- }
- }
-#endif
-}
-
-
-size_t ZSTD_compressContinue(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- const BYTE* const istart = (const BYTE* const)src;
- const BYTE* ip = istart;
- BYTE* const ostart = (BYTE* const)dst;
- BYTE* op = ostart;
- const U32 updateRate = 2 * BLOCKSIZE;
-
- /* Init */
- if (ctx->base==NULL)
- ctx->base = (const BYTE*)src, ctx->current=0, ctx->nextUpdate = g_maxDistance;
- if (src != ctx->base + ctx->current) /* not contiguous */
- {
- ZSTD_resetCCtx(ctx);
- ctx->base = (const BYTE*)src;
- ctx->current = 0;
- }
- ctx->current += (U32)srcSize;
-
- while (srcSize)
- {
- size_t cSize;
- size_t blockSize = BLOCKSIZE;
- if (blockSize > srcSize) blockSize = srcSize;
-
- if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */
- return ERROR(dstSize_tooSmall);
-
- /* update hash table */
- if (g_maxDistance <= BLOCKSIZE) /* static test ; yes == blocks are independent */
- {
- ZSTD_resetCCtx(ctx);
- ctx->base = ip;
- ctx->current=0;
- }
- else if (ip >= ctx->base + ctx->nextUpdate)
- {
- ctx->nextUpdate += updateRate;
- ZSTD_limitCtx(ctx, ctx->nextUpdate - g_maxDistance);
- }
-
- /* compress */
- cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
- if (cSize == 0)
- {
- cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
- if (ZSTD_isError(cSize)) return cSize;
- }
- else
- {
- if (ZSTD_isError(cSize)) return cSize;
- op[0] = (BYTE)(cSize>>16);
- op[1] = (BYTE)(cSize>>8);
- op[2] = (BYTE)cSize;
- op[0] += (BYTE)(bt_compressed << 6); /* is a compressed block */
- cSize += 3;
- }
- op += cSize;
- maxDstSize -= cSize;
- ip += blockSize;
- srcSize -= blockSize;
- }
-
- return op-ostart;
-}
-
-
-size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
-{
- BYTE* op = (BYTE*)dst;
-
- /* Sanity check */
- (void)ctx;
- if (maxDstSize < ZSTD_blockHeaderSize) return ERROR(dstSize_tooSmall);
-
- /* End of frame */
- op[0] = (BYTE)(bt_end << 6);
- op[1] = 0;
- op[2] = 0;
-
- return 3;
-}
-
-
-size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- BYTE* const ostart = (BYTE* const)dst;
- BYTE* op = ostart;
- size_t oSize;
-
- /* Header */
- oSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
- if(ZSTD_isError(oSize)) return oSize;
- op += oSize;
- maxDstSize -= oSize;
-
- /* Compression */
- oSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
- if (ZSTD_isError(oSize)) return oSize;
- op += oSize;
- maxDstSize -= oSize;
-
- /* Close frame */
- oSize = ZSTD_compressEnd(ctx, op, maxDstSize);
- if(ZSTD_isError(oSize)) return oSize;
- op += oSize;
-
- return (op - ostart);
-}
-
-
-size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- size_t r;
-#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
- ZSTD_CCtx* ctx;
- ctx = ZSTD_createCCtx();
- if (ctx==NULL) return ERROR(GENERIC);
-# else
- ZSTD_CCtx ctxBody;
- ZSTD_CCtx* const ctx = &ctxBody;
-# endif
-
- r = ZSTD_compressCCtx(ctx, dst, maxDstSize, src, srcSize);
-
-#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
- ZSTD_freeCCtx(ctx);
-#endif
-
- return r;
-}
-
-
-/* *************************************************************
-* Decompression section
-***************************************************************/
-struct ZSTD_DCtx_s
-{
- U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
- U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
- U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
- void* previousDstEnd;
- void* base;
- size_t expected;
- blockType_t bType;
- U32 phase;
- const BYTE* litPtr;
- size_t litBufSize;
- size_t litSize;
- BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
-}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
-
-
-size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
-{
- const BYTE* const in = (const BYTE* const)src;
- BYTE headerFlags;
- U32 cSize;
-
- if (srcSize < 3) return ERROR(srcSize_wrong);
-
- headerFlags = *in;
- cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16);
-
- bpPtr->blockType = (blockType_t)(headerFlags >> 6);
- bpPtr->origSize = (bpPtr->blockType == bt_rle) ? cSize : 0;
-
- if (bpPtr->blockType == bt_end) return 0;
- if (bpPtr->blockType == bt_rle) return 1;
- return cSize;
-}
-
-static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
- memcpy(dst, src, srcSize);
- return srcSize;
-}
-
-
-/** ZSTD_decompressLiterals
- @return : nb of bytes read from src, or an error code*/
-static size_t ZSTD_decompressLiterals(void* dst, size_t* maxDstSizePtr,
- const void* src, size_t srcSize)
-{
- const BYTE* ip = (const BYTE*)src;
-
- const size_t litSize = (MEM_readLE32(src) & 0x1FFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
- const size_t litCSize = (MEM_readLE32(ip+2) & 0xFFFFFF) >> 5; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
-
- if (litSize > *maxDstSizePtr) return ERROR(corruption_detected);
- if (litCSize + 5 > srcSize) return ERROR(corruption_detected);
-
- if (HUF_isError(HUF_decompress(dst, litSize, ip+5, litCSize))) return ERROR(corruption_detected);
-
- *maxDstSizePtr = litSize;
- return litCSize + 5;
-}
-
-
-/** ZSTD_decodeLiteralsBlock
- @return : nb of bytes read from src (< srcSize )*/
-size_t ZSTD_decodeLiteralsBlock(void* ctx,
- const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
-{
- ZSTD_DCtx* dctx = (ZSTD_DCtx*)ctx;
- const BYTE* const istart = (const BYTE*) src;
-
- /* any compressed block with literals segment must be at least this size */
- if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
-
- switch(*istart & 3)
- {
- /* compressed */
- case 0:
- {
- size_t litSize = BLOCKSIZE;
- const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
- dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
- dctx->litSize = litSize;
- return readSize; /* works if it's an error too */
- }
- case IS_RAW:
- {
- const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
- if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
- {
- if (litSize > srcSize-3) return ERROR(corruption_detected);
- memcpy(dctx->litBuffer, istart, litSize);
- dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
- dctx->litSize = litSize;
- return litSize+3;
- }
- /* direct reference into compressed stream */
- dctx->litPtr = istart+3;
- dctx->litBufSize = srcSize-3;
- dctx->litSize = litSize;
- return litSize+3; }
- case IS_RLE:
- {
- const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
- if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
- memset(dctx->litBuffer, istart[3], litSize);
- dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
- dctx->litSize = litSize;
- return 4;
- }
- default:
- return ERROR(corruption_detected); /* forbidden nominal case */
- }
-}
-
-
-size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
- FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
- const void* src, size_t srcSize)
-{
- const BYTE* const istart = (const BYTE* const)src;
- const BYTE* ip = istart;
- const BYTE* const iend = istart + srcSize;
- U32 LLtype, Offtype, MLtype;
- U32 LLlog, Offlog, MLlog;
- size_t dumpsLength;
-
- /* check */
- if (srcSize < 5) return ERROR(srcSize_wrong);
-
- /* SeqHead */
- *nbSeq = MEM_readLE16(ip); ip+=2;
- LLtype = *ip >> 6;
- Offtype = (*ip >> 4) & 3;
- MLtype = (*ip >> 2) & 3;
- if (*ip & 2)
- {
- dumpsLength = ip[2];
- dumpsLength += ip[1] << 8;
- ip += 3;
- }
- else
- {
- dumpsLength = ip[1];
- dumpsLength += (ip[0] & 1) << 8;
- ip += 2;
- }
- *dumpsPtr = ip;
- ip += dumpsLength;
- *dumpsLengthPtr = dumpsLength;
-
- /* check */
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
-
- /* sequences */
- {
- S16 norm[MaxML+1]; /* assumption : MaxML >= MaxLL and MaxOff */
- size_t headerSize;
-
- /* Build DTables */
- switch(LLtype)
- {
- U32 max;
- case bt_rle :
- LLlog = 0;
- FSE_buildDTable_rle(DTableLL, *ip++); break;
- case bt_raw :
- LLlog = LLbits;
- FSE_buildDTable_raw(DTableLL, LLbits); break;
- default :
- max = MaxLL;
- headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
- if (FSE_isError(headerSize)) return ERROR(GENERIC);
- if (LLlog > LLFSELog) return ERROR(corruption_detected);
- ip += headerSize;
- FSE_buildDTable(DTableLL, norm, max, LLlog);
- }
-
- switch(Offtype)
- {
- U32 max;
- case bt_rle :
- Offlog = 0;
- if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
- FSE_buildDTable_rle(DTableOffb, *ip++ & MaxOff); /* if *ip > MaxOff, data is corrupted */
- break;
- case bt_raw :
- Offlog = Offbits;
- FSE_buildDTable_raw(DTableOffb, Offbits); break;
- default :
- max = MaxOff;
- headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
- if (FSE_isError(headerSize)) return ERROR(GENERIC);
- if (Offlog > OffFSELog) return ERROR(corruption_detected);
- ip += headerSize;
- FSE_buildDTable(DTableOffb, norm, max, Offlog);
- }
-
- switch(MLtype)
- {
- U32 max;
- case bt_rle :
- MLlog = 0;
- if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
- FSE_buildDTable_rle(DTableML, *ip++); break;
- case bt_raw :
- MLlog = MLbits;
- FSE_buildDTable_raw(DTableML, MLbits); break;
- default :
- max = MaxML;
- headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
- if (FSE_isError(headerSize)) return ERROR(GENERIC);
- if (MLlog > MLFSELog) return ERROR(corruption_detected);
- ip += headerSize;
- FSE_buildDTable(DTableML, norm, max, MLlog);
- }
- }
-
- return ip-istart;
-}
-
-
-typedef struct {
- size_t litLength;
- size_t offset;
- size_t matchLength;
-} seq_t;
-
-typedef struct {
- BIT_DStream_t DStream;
- FSE_DState_t stateLL;
- FSE_DState_t stateOffb;
- FSE_DState_t stateML;
- size_t prevOffset;
- const BYTE* dumps;
- const BYTE* dumpsEnd;
-} seqState_t;
-
-
-static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
-{
- size_t litLength;
- size_t prevOffset;
- size_t offset;
- size_t matchLength;
- const BYTE* dumps = seqState->dumps;
- const BYTE* const de = seqState->dumpsEnd;
-
- /* Literal length */
- litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
- prevOffset = litLength ? seq->offset : seqState->prevOffset;
- seqState->prevOffset = seq->offset;
- if (litLength == MaxLL)
- {
- U32 add = *dumps++;
- if (add < 255) litLength += add;
- else
- {
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
- dumps += 3;
- }
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
- }
-
- /* Offset */
- {
- static const U32 offsetPrefix[MaxOff+1] = {
- 1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
- 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
- 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
- U32 offsetCode, nbBits;
- offsetCode = FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream)); /* <= maxOff, by table construction */
- if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
- nbBits = offsetCode - 1;
- if (offsetCode==0) nbBits = 0; /* cmove */
- offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits);
- if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
- if (offsetCode==0) offset = prevOffset; /* cmove */
- }
-
- /* MatchLength */
- matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
- if (matchLength == MaxML)
- {
- U32 add = *dumps++;
- if (add < 255) matchLength += add;
- else
- {
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
- dumps += 3;
- }
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
- }
- matchLength += MINMATCH;
-
- /* save result */
- seq->litLength = litLength;
- seq->offset = offset;
- seq->matchLength = matchLength;
- seqState->dumps = dumps;
-}
-
-
-static size_t ZSTD_execSequence(BYTE* op,
- seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit_8,
- BYTE* const base, BYTE* const oend)
-{
- static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
- const BYTE* const ostart = op;
- BYTE* const oLitEnd = op + sequence.litLength;
- BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
- BYTE* const oend_8 = oend-8;
- const BYTE* const litEnd = *litPtr + sequence.litLength;
-
- /* check */
- if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
- if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
- if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
-
- /* copy Literals */
- ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
- op = oLitEnd;
- *litPtr = litEnd; /* update for next sequence */
-
- /* copy Match */
- {
- const BYTE* match = op - sequence.offset;
-
- /* check */
- //if (match > op) return ERROR(corruption_detected); /* address space overflow test (is clang optimizer removing this test ?) */
- if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */
- if (match < base) return ERROR(corruption_detected);
-
- /* close range match, overlap */
- if (sequence.offset < 8)
- {
- const int dec64 = dec64table[sequence.offset];
- op[0] = match[0];
- op[1] = match[1];
- op[2] = match[2];
- op[3] = match[3];
- match += dec32table[sequence.offset];
- ZSTD_copy4(op+4, match);
- match -= dec64;
- }
- else
- {
- ZSTD_copy8(op, match);
- }
- op += 8; match += 8;
-
- if (oMatchEnd > oend-12)
- {
- if (op < oend_8)
- {
- ZSTD_wildcopy(op, match, oend_8 - op);
- match += oend_8 - op;
- op = oend_8;
- }
- while (op < oMatchEnd) *op++ = *match++;
- }
- else
- {
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
- }
- }
-
- return oMatchEnd - ostart;
-}
-
-static size_t ZSTD_decompressSequences(
- void* ctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize)
-{
- ZSTD_DCtx* dctx = (ZSTD_DCtx*)ctx;
- const BYTE* ip = (const BYTE*)seqStart;
- const BYTE* const iend = ip + seqSize;
- BYTE* const ostart = (BYTE* const)dst;
- BYTE* op = ostart;
- BYTE* const oend = ostart + maxDstSize;
- size_t errorCode, dumpsLength;
- const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
- const BYTE* const litEnd = litPtr + dctx->litSize;
- int nbSeq;
- const BYTE* dumps;
- U32* DTableLL = dctx->LLTable;
- U32* DTableML = dctx->MLTable;
- U32* DTableOffb = dctx->OffTable;
- BYTE* const base = (BYTE*) (dctx->base);
-
- /* Build Decoding Tables */
- errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
- DTableLL, DTableML, DTableOffb,
- ip, iend-ip);
- if (ZSTD_isError(errorCode)) return errorCode;
- ip += errorCode;
-
- /* Regen sequences */
- {
- seq_t sequence;
- seqState_t seqState;
-
- memset(&sequence, 0, sizeof(sequence));
- sequence.offset = 4;
- seqState.dumps = dumps;
- seqState.dumpsEnd = dumps + dumpsLength;
- seqState.prevOffset = 4;
- errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
- if (ERR_isError(errorCode)) return ERROR(corruption_detected);
- FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
- FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
- FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
-
- for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (nbSeq>0) ; )
- {
- size_t oneSeqSize;
- nbSeq--;
- ZSTD_decodeSequence(&sequence, &seqState);
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litLimit_8, base, oend);
- if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
- op += oneSeqSize;
- }
-
- /* check if reached exact end */
- if ( !BIT_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected); /* requested too much : data is corrupted */
- if (nbSeq<0) return ERROR(corruption_detected); /* requested too many sequences : data is corrupted */
-
- /* last literal segment */
- {
- size_t lastLLSize = litEnd - litPtr;
- if (litPtr > litEnd) return ERROR(corruption_detected);
- if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
- if (op != litPtr) memcpy(op, litPtr, lastLLSize);
- op += lastLLSize;
- }
- }
-
- return op-ostart;
-}
-
-
-static size_t ZSTD_decompressBlock(
- void* ctx,
- void* dst, size_t maxDstSize,
- const void* src, size_t srcSize)
-{
- /* blockType == blockCompressed */
- const BYTE* ip = (const BYTE*)src;
-
- /* Decode literals sub-block */
- size_t litCSize = ZSTD_decodeLiteralsBlock(ctx, src, srcSize);
- if (ZSTD_isError(litCSize)) return litCSize;
- ip += litCSize;
- srcSize -= litCSize;
-
- return ZSTD_decompressSequences(ctx, dst, maxDstSize, ip, srcSize);
-}
-
-
-static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- const BYTE* ip = (const BYTE*)src;
- const BYTE* iend = ip + srcSize;
- BYTE* const ostart = (BYTE* const)dst;
- BYTE* op = ostart;
- BYTE* const oend = ostart + maxDstSize;
- size_t remainingSize = srcSize;
- U32 magicNumber;
- blockProperties_t blockProperties;
-
- /* Frame Header */
- if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
- magicNumber = MEM_readLE32(src);
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
- if (ZSTD_isLegacy(magicNumber))
- return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber);
-#endif
- if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
- ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
-
- /* Loop on each block */
- while (1)
- {
- size_t decodedSize=0;
- size_t cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
-
- ip += ZSTD_blockHeaderSize;
- remainingSize -= ZSTD_blockHeaderSize;
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
-
- switch(blockProperties.blockType)
- {
- case bt_compressed:
- decodedSize = ZSTD_decompressBlock(ctx, op, oend-op, ip, cBlockSize);
- break;
- case bt_raw :
- decodedSize = ZSTD_copyUncompressedBlock(op, oend-op, ip, cBlockSize);
- break;
- case bt_rle :
- return ERROR(GENERIC); /* not yet supported */
- break;
- case bt_end :
- /* end of frame */
- if (remainingSize) return ERROR(srcSize_wrong);
- break;
- default:
- return ERROR(GENERIC); /* impossible */
- }
- if (cBlockSize == 0) break; /* bt_end */
-
- if (ZSTD_isError(decodedSize)) return decodedSize;
- op += decodedSize;
- ip += cBlockSize;
- remainingSize -= cBlockSize;
- }
-
- return op-ostart;
-}
-
-size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- ZSTD_DCtx ctx;
- ctx.base = dst;
- return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
-}
-
-
-/* ******************************
-* Streaming Decompression API
-********************************/
-
-size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
-{
- dctx->expected = ZSTD_frameHeaderSize;
- dctx->phase = 0;
- dctx->previousDstEnd = NULL;
- dctx->base = NULL;
- return 0;
-}
-
-ZSTD_DCtx* ZSTD_createDCtx(void)
-{
- ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx));
- if (dctx==NULL) return NULL;
- ZSTD_resetDCtx(dctx);
- return dctx;
-}
-
-size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
-{
- free(dctx);
- return 0;
-}
-
-size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
-{
- return dctx->expected;
-}
-
-size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
-{
- /* Sanity check */
- if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
- if (dst != ctx->previousDstEnd) /* not contiguous */
- ctx->base = dst;
-
- /* Decompress : frame header */
- if (ctx->phase == 0)
- {
- /* Check frame magic header */
- U32 magicNumber = MEM_readLE32(src);
- if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
- ctx->phase = 1;
- ctx->expected = ZSTD_blockHeaderSize;
- return 0;
- }
-
- /* Decompress : block header */
- if (ctx->phase == 1)
- {
- blockProperties_t bp;
- size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
- if (ZSTD_isError(blockSize)) return blockSize;
- if (bp.blockType == bt_end)
- {
- ctx->expected = 0;
- ctx->phase = 0;
- }
- else
- {
- ctx->expected = blockSize;
- ctx->bType = bp.blockType;
- ctx->phase = 2;
- }
-
- return 0;
- }
-
- /* Decompress : block content */
- {
- size_t rSize;
- switch(ctx->bType)
- {
- case bt_compressed:
- rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
- break;
- case bt_raw :
- rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize);
- break;
- case bt_rle :
- return ERROR(GENERIC); /* not yet handled */
- break;
- case bt_end : /* should never happen (filtered at phase 1) */
- rSize = 0;
- break;
- default:
- return ERROR(GENERIC);
- }
- ctx->phase = 1;
- ctx->expected = ZSTD_blockHeaderSize;
- ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
- return rSize;
- }
-
-}
-
-
diff --git a/lib/zstd.h b/lib/zstd.h
index 96bb184..d6eb0b5 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -43,25 +43,39 @@ extern "C" {
#include <stddef.h> /* size_t */
+/* ***************************************************************
+* Tuning parameters
+*****************************************************************/
+/*!
+* ZSTD_DLL_EXPORT :
+* Enable exporting of functions when building a Windows DLL
+*/
+#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDLIB_API __declspec(dllexport)
+#else
+# define ZSTDLIB_API
+#endif
+
+
/* *************************************
* Version
***************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
-unsigned ZSTD_versionNumber (void);
+ZSTDLIB_API unsigned ZSTD_versionNumber (void);
/* *************************************
* Simple functions
***************************************/
-size_t ZSTD_compress( void* dst, size_t maxDstSize,
- const void* src, size_t srcSize,
- int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize,
+ int compressionLevel);
-size_t ZSTD_decompress( void* dst, size_t maxOriginalSize,
- const void* src, size_t compressedSize);
+ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t maxOriginalSize,
+ const void* src, size_t compressedSize);
/**
ZSTD_compress() :
@@ -83,25 +97,25 @@ ZSTD_decompress() :
/* *************************************
* Tool functions
***************************************/
-size_t ZSTD_compressBound(size_t srcSize); /** maximum compressed size (worst case scenario) */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /** maximum compressed size (worst case scenario) */
/* Error Management */
-unsigned ZSTD_isError(size_t code); /** tells if a return value is an error code */
-const char* ZSTD_getErrorName(size_t code); /** provides error code string */
+ZSTDLIB_API unsigned ZSTD_isError(size_t code); /** tells if a return value is an error code */
+ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /** provides error code string */
/* *************************************
* Advanced functions
***************************************/
typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */
-ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
+ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
+ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
/**
ZSTD_compressCCtx() :
Same as ZSTD_compress(), but requires a ZSTD_CCtx working space already allocated
*/
-size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
#if defined (__cplusplus)
diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c
index 844c504..99148c2 100644
--- a/lib/zstd_buffered.c
+++ b/lib/zstd_buffered.c
@@ -160,6 +160,12 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
}
+ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* zbc, const void* src, size_t srcSize)
+{
+ ZSTD_compress_insertDictionary(zbc->zc, src, srcSize);
+ return 0;
+}
+
/* *** Compression *** */
@@ -327,6 +333,8 @@ struct ZBUFF_DCtx_s {
size_t outStart;
size_t outEnd;
size_t hPos;
+ const char* dict;
+ size_t dictSize;
ZBUFF_dStage stage;
unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
@@ -353,17 +361,23 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
}
-
/* *** Initialization *** */
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
{
zbc->stage = ZBUFFds_readHeader;
- zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
+ zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = zbc->dictSize = 0;
return ZSTD_resetDCtx(zbc->zc);
}
+size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* zbc, const void* src, size_t srcSize)
+{
+ zbc->dict = (const char*)src;
+ zbc->dictSize = srcSize;
+ return 0;
+}
+
/* *** Decompression *** */
@@ -442,6 +456,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
}
}
+ if (zbc->dictSize)
+ ZSTD_decompress_insertDictionary(zbc->zc, zbc->dict, zbc->dictSize);
if (zbc->hPos)
{
/* some data already loaded into headerBuffer : transfer into inBuff */
diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h
index 80ba819..d2316a8 100644
--- a/lib/zstd_buffered.h
+++ b/lib/zstd_buffered.h
@@ -47,17 +47,32 @@ extern "C" {
#include <stddef.h> /* size_t */
+/* ***************************************************************
+* Tuning parameters
+*****************************************************************/
+/*!
+* ZSTD_DLL_EXPORT :
+* Enable exporting of functions when building a Windows DLL
+*/
+#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDLIB_API __declspec(dllexport)
+#else
+# define ZSTDLIB_API
+#endif
+
+
/* *************************************
* Streaming functions
***************************************/
typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
-ZBUFF_CCtx* ZBUFF_createCCtx(void);
-size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
+ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
+ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
-size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
-size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
-size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
-size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
+ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
+ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* cctx, const void* src, size_t srcSize);
+ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
+ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
+ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
/** ************************************************
* Streaming compression
@@ -67,6 +82,9 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
* Use ZBUFF_compressInit() to start a new compression operation.
* ZBUFF_CCtx objects can be reused multiple times.
*
+* Optionally, a reference to a static dictionary can be created with ZBUFF_compressWithDictionary()
+* Note that the dictionary content must remain accessible during the compression process.
+*
* Use ZBUFF_compressContinue() repetitively to consume input stream.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr.
@@ -97,11 +115,13 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
-ZBUFF_DCtx* ZBUFF_createDCtx(void);
-size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
+ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
+ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
+
+ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
+ZSTDLIB_API size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* dctx, const void* src, size_t srcSize);
-size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
-size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
+ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
/** ************************************************
* Streaming decompression
@@ -111,6 +131,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizeP
* Use ZBUFF_decompressInit() to start a new decompression operation.
* ZBUFF_DCtx objects can be reused multiple times.
*
+* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary()
+* It must be the same content as the one set during compression phase.
+* Dictionary content must remain accessible during the decompression process.
+*
* Use ZBUFF_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
@@ -129,15 +153,15 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizeP
/* *************************************
* Tool functions
***************************************/
-unsigned ZBUFF_isError(size_t errorCode);
-const char* ZBUFF_getErrorName(size_t errorCode);
+ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
+ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
/** The below functions provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are not compulsory, they just tend to offer better latency */
-size_t ZBUFF_recommendedCInSize(void);
-size_t ZBUFF_recommendedCOutSize(void);
-size_t ZBUFF_recommendedDInSize(void);
-size_t ZBUFF_recommendedDOutSize(void);
+ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
+ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);
+ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
+ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
#if defined (__cplusplus)
diff --git a/lib/zstd_buffered_static.h b/lib/zstd_buffered_static.h
index cba1d6f..7d9ee27 100644
--- a/lib/zstd_buffered_static.h
+++ b/lib/zstd_buffered_static.h
@@ -52,7 +52,7 @@ extern "C" {
/* *************************************
* Advanced Streaming functions
***************************************/
-size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params);
+ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params);
#if defined (__cplusplus)
diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c
index 5ca4437..c6d81eb 100644
--- a/lib/zstd_compress.c
+++ b/lib/zstd_compress.c
@@ -65,6 +65,7 @@
/* *************************************
* Constants
***************************************/
+unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const U32 g_searchStrength = 8;
@@ -100,8 +101,6 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
/* *************************************
* Context memory management
***************************************/
-#define WORKPLACESIZE (BLOCKSIZE*3)
-
struct ZSTD_CCtx_s
{
const BYTE* nextSrc; /* next block here to continue on current prefix */
@@ -113,6 +112,7 @@ struct ZSTD_CCtx_s
ZSTD_parameters params;
void* workSpace;
size_t workSpaceSize;
+ size_t blockSize;
seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable;
@@ -170,11 +170,14 @@ void ZSTD_validateParams(ZSTD_parameters* params)
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
ZSTD_parameters params)
{
+ /* note : params considered validated here */
+ const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog);
+
/* reserve table memory */
{
const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog)) * sizeof(U32);
- const size_t neededSpace = tableSpace + WORKPLACESIZE;
+ const size_t neededSpace = tableSpace + (3*blockSize);
if (zc->workSpaceSize < neededSpace)
{
free(zc->workSpace);
@@ -195,12 +198,13 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->dictLimit = 0;
zc->lowLimit = 0;
zc->params = params;
+ zc->blockSize = blockSize;
zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
- zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (BLOCKSIZE>>2));
- zc->seqStore.litStart = zc->seqStore.offCodeStart + (BLOCKSIZE>>2);
- zc->seqStore.litLengthStart = zc->seqStore.litStart + BLOCKSIZE;
- zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (BLOCKSIZE>>2);
- zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (BLOCKSIZE>>2);
+ zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (blockSize>>2));
+ zc->seqStore.litStart = zc->seqStore.offCodeStart + (blockSize>>2);
+ zc->seqStore.litLengthStart = zc->seqStore.litStart + blockSize;
+ zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2);
+ zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
return 0;
}
@@ -485,7 +489,7 @@ size_t ZSTD_compressSequences(void* dst, size_t maxDstSize,
BYTE litLength = llTable[i]; /* (7)*/ /* (7)*/
FSE_encodeSymbol(&blockStream, &stateMatchLength, matchLength); /* 17 */ /* 17 */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
- BIT_addBits(&blockStream, offset, nbBits); /* 32 */ /* 42 */
+ BIT_addBits(&blockStream, offset, nbBits); /* 31 */ /* 42 */ /* 24 bits max in 32-bits mode */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 16 */ /* 51 */
FSE_encodeSymbol(&blockStream, &stateLitLength, litLength); /* 26 */ /* 61 */
@@ -596,7 +600,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
- return (int)(r>>3);
+ return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
# else
@@ -609,7 +613,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
# if defined(_MSC_VER)
unsigned long r=0;
_BitScanForward( &r, (U32)val );
- return (int)(r>>3);
+ return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
@@ -620,7 +624,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
}
else /* Big Endian CPU */
{
- if (MEM_32bits())
+ if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
@@ -727,13 +731,30 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
* Fast Scan
***************************************/
+#define FILLHASHSTEP 3
+static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
+{
+ U32* const hashTable = zc->hashTable;
+ const U32 hBits = zc->params.hashLog;
+ const BYTE* const base = zc->base;
+ const BYTE* ip = base + zc->nextToUpdate;
+ const BYTE* const iend = (const BYTE*) end;
+
+ while(ip <= iend)
+ {
+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
+ ip += FILLHASHSTEP;
+ }
+}
+
+
FORCE_INLINE
size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
const U32 mls)
{
- U32* hashTable = zc->hashTable;
+ U32* const hashTable = zc->hashTable;
const U32 hBits = zc->params.hashLog;
seqStore_t* seqStorePtr = &(zc->seqStore);
const BYTE* const base = zc->base;
@@ -749,12 +770,12 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
/* init */
ZSTD_resetSeqStore(seqStorePtr);
- if (ip < base+4)
+ if (ip < lowest+4)
{
- hashTable[ZSTD_hashPtr(base+1, hBits, mls)] = 1;
- hashTable[ZSTD_hashPtr(base+2, hBits, mls)] = 2;
- hashTable[ZSTD_hashPtr(base+3, hBits, mls)] = 3;
- ip = base+4;
+ hashTable[ZSTD_hashPtr(lowest+1, hBits, mls)] = zc->dictLimit+1;
+ hashTable[ZSTD_hashPtr(lowest+2, hBits, mls)] = zc->dictLimit+2;
+ hashTable[ZSTD_hashPtr(lowest+3, hBits, mls)] = zc->dictLimit+3;
+ ip = lowest+4;
}
/* Main Search Loop */
@@ -1005,7 +1026,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const base = zc->base;
const BYTE* match = base + matchIndex;
- U32 current = (U32)(ip-base);
+ const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
@@ -1013,13 +1034,14 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
const U32 windowLow = zc->lowLimit;
if ( (current-matchIndex == 1) /* RLE */
+ && (matchIndex > windowLow)
&& (MEM_read64(match) == MEM_read64(ip)) )
{
size_t rleLength = ZSTD_count(ip+8, match+8, iend) + 8;
return (U32)(rleLength - mls);
}
- hashTable[h] = (U32)(ip - base); /* Update Hash Table */
+ hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow))
{
@@ -1082,7 +1104,7 @@ size_t ZSTD_insertBtAndFindBestMatch (
size_t bestLength = 0;
U32 dummy32; /* to be nullified at the end */
- hashTable[h] = (U32)(ip-base); /* Update Hash Table */
+ hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow))
{
@@ -1192,7 +1214,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* match = base + matchIndex;
- U32 current = (U32)(ip-base);
+ const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
@@ -1206,7 +1228,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32
return (U32)(rleLength - mls);
}
- hashTable[h] = (U32)(ip - base); /* Update Hash Table */
+ hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow))
{
@@ -1297,7 +1319,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict (
size_t bestLength = 0;
U32 dummy32; /* to be nullified at the end */
- hashTable[h] = (U32)(ip-base); /* Update Hash Table */
+ hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow))
{
@@ -1515,6 +1537,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
+ const BYTE* const base = ctx->base + ctx->dictLimit;
size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
const U32 maxSearches = 1 << ctx->params.searchLog;
@@ -1527,7 +1550,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* init */
ZSTD_resetSeqStore(seqStorePtr);
- if (((ip-ctx->base) - ctx->dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
+ if ((ip-base) < REPCODE_STARTVALUE) ip = base + REPCODE_STARTVALUE;
/* Match Loop */
while (ip < ilimit)
@@ -1552,7 +1575,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
matchLength = ml2, start = ip, offset=offsetFound;
}
- if (matchLength < MINMATCH)
+ if (matchLength < MINMATCH)
{
ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
continue;
@@ -1613,7 +1636,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* catch up */
if (offset)
{
- while ((start>anchor) && (start>ctx->base+offset) && (start[-1] == start[-1-offset])) /* only search for offset within prefix */
+ while ((start>anchor) && (start>base+offset) && (start[-1] == start[-1-offset])) /* only search for offset within prefix */
{ start--; matchLength++; }
offset_2 = offset_1; offset_1 = offset;
}
@@ -1704,7 +1727,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* init */
ZSTD_resetSeqStore(seqStorePtr);
- if (((ip-base) - dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
+ if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
/* Match Loop */
while (ip < ilimit)
@@ -1919,7 +1942,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
- size_t blockSize = BLOCKSIZE;
+ size_t blockSize = ctxPtr->blockSize;
size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
@@ -1972,28 +1995,33 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
{
const BYTE* const ip = (const BYTE*) src;
- /* preemptive overflow correction */
- if ((zc->base > (const BYTE*)src) || (zc->lowLimit > (1<<30) ))
- {
- U32 correction = zc->lowLimit-1;
- ZSTD_reduceIndex(zc, correction);
- zc->base += correction;
- zc->dictBase += correction;
- zc->lowLimit -= correction;
- zc->dictLimit -= correction;
- if (zc->nextToUpdate < correction) zc->nextToUpdate = 0;
- else zc->nextToUpdate -= correction;
- }
-
/* Check if blocks follow each other */
if (src != zc->nextSrc)
{
/* not contiguous */
+ size_t delta = zc->nextSrc - ip;
zc->lowLimit = zc->dictLimit;
zc->dictLimit = (U32)(zc->nextSrc - zc->base);
zc->dictBase = zc->base;
- zc->base += ip - zc->nextSrc;
+ zc->base -= delta;
zc->nextToUpdate = zc->dictLimit;
+ if (zc->dictLimit - zc->lowLimit < 8) zc->lowLimit = zc->dictLimit; /* too small extDict */
+ }
+
+ /* preemptive overflow correction */
+ if (zc->lowLimit > (1<<30))
+ {
+ U32 btplus = (zc->params.strategy == ZSTD_btlazy2);
+ U32 contentMask = (1 << (zc->params.contentLog - btplus)) - 1;
+ U32 newLowLimit = zc->lowLimit & contentMask; /* preserve position % contentSize */
+ U32 correction = zc->lowLimit - newLowLimit;
+ ZSTD_reduceIndex(zc, correction);
+ zc->base += correction;
+ zc->dictBase += correction;
+ zc->lowLimit = newLowLimit;
+ zc->dictLimit -= correction;
+ if (zc->nextToUpdate < correction) zc->nextToUpdate = 0;
+ else zc->nextToUpdate -= correction;
}
/* input-dictionary overlap */
@@ -2008,8 +2036,46 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
return ZSTD_compress_generic (zc, dst, dstSize, src, srcSize);
}
+size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize)
+{
+ const BYTE* const ip = (const BYTE*) src;
+ const BYTE* const iend = ip + srcSize;
+
+ /* input becomes current prefix */
+ zc->lowLimit = zc->dictLimit;
+ zc->dictLimit = (U32)(zc->nextSrc - zc->base);
+ zc->dictBase = zc->base;
+ zc->base += ip - zc->nextSrc;
+ zc->nextToUpdate = zc->dictLimit;
+
+ zc->nextSrc = iend;
+ if (srcSize <= 8) return 0;
+
+ switch(zc->params.strategy)
+ {
+ case ZSTD_fast:
+ ZSTD_fillHashTable (zc, iend-8, zc->params.searchLength);
+ break;
+
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ ZSTD_insertAndFindFirstIndex (zc, iend-8, zc->params.searchLength);
+ break;
+
+ case ZSTD_btlazy2:
+ ZSTD_updateTree(zc, iend-8, iend, 1 << zc->params.searchLog, zc->params.searchLength);
+ break;
+
+ default:
+ return ERROR(GENERIC); /* strategy doesn't exist; impossible */
+ }
+
+ return 0;
+}
+
-/** ZSTD_compressBegin_advanced
+/*! ZSTD_compressBegin_advanced
* Write frame header, according to params
* @return : nb of bytes written */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx,
@@ -2036,7 +2102,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx,
ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint)
{
ZSTD_parameters result;
- int tableID = ((srcSizeHint-1) <= 128 KB); /* intentional underflow for srcSizeHint == 0 */
+ int tableID = ((srcSizeHint-1) <= 256 KB) + ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
result = ZSTD_defaultParameters[tableID][compressionLevel];
@@ -2051,7 +2117,7 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int comp
}
-/** ZSTD_compressEnd
+/*! ZSTD_compressEnd
* Write frame epilogue
* @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
@@ -2073,6 +2139,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
ZSTD_parameters params)
{
BYTE* const ostart = (BYTE*)dst;
@@ -2085,9 +2152,15 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
op += oSize;
maxDstSize -= oSize;
+ /* dictionary */
+ if (dict)
+ {
+ oSize = ZSTD_compress_insertDictionary(ctx, dict, dictSize);
+ if (ZSTD_isError(oSize)) return oSize;
+ }
+
/* body (compression) */
- ctx->base = (const BYTE*)src;
- oSize = ZSTD_compress_generic (ctx, op, maxDstSize, src, srcSize);
+ oSize = ZSTD_compressContinue (ctx, op, maxDstSize, src, srcSize);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;
@@ -2100,9 +2173,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
return (op - ostart);
}
+size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
+{
+ return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize+dictSize));
+}
+
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{
- return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_getParams(compressionLevel, srcSize));
+ return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize));
}
size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
@@ -2114,3 +2192,4 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi
free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; free heap content */
return result;
}
+
diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c
index e8190bc..3431e32 100644
--- a/lib/zstd_decompress.c
+++ b/lib/zstd_decompress.c
@@ -127,10 +127,10 @@ struct ZSTD_DCtx_s
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
- void* previousDstEnd;
- void* base;
- void* vBase;
- void* dictEnd;
+ const void* previousDstEnd;
+ const void* base;
+ const void* vBase;
+ const void* dictEnd;
size_t expected;
size_t headerSize;
ZSTD_parameters params;
@@ -141,7 +141,7 @@ struct ZSTD_DCtx_s
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
-}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
+}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
{
@@ -505,7 +505,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8,
- BYTE* const base, BYTE* const vBase, BYTE* const dictEnd)
+ const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
@@ -527,64 +527,61 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
*litPtr = litEnd; /* update for next sequence */
/* copy Match */
- /* check */
- //if (match > oLitEnd) return ERROR(corruption_detected); /* address space overflow test (is clang optimizer wrongly removing this test ?) */
- if (sequence.offset > (size_t)oLitEnd) return ERROR(corruption_detected); /* address space overflow test (this test seems preserved by clang optimizer) */
-
- if (match < base)
- {
- /* offset beyond prefix */
- if (match < vBase) return ERROR(corruption_detected);
- match = dictEnd - (base-match);
- if (match + sequence.matchLength <= dictEnd)
- {
- memcpy(oLitEnd, match, sequence.matchLength);
- return sequenceLength;
- }
- /* span extDict & currentPrefixSegment */
- {
- size_t length1 = dictEnd - match;
- memcpy(oLitEnd, match, length1);
- op = oLitEnd + length1;
- sequence.matchLength -= length1;
- match = base;
- }
- }
-
- /* match within prefix */
- if (sequence.offset < 8)
- {
- /* close range match, overlap */
- const int sub2 = dec64table[sequence.offset];
- op[0] = match[0];
- op[1] = match[1];
- op[2] = match[2];
- op[3] = match[3];
- match += dec32table[sequence.offset];
- ZSTD_copy4(op+4, match);
- match -= sub2;
- }
- else
- {
- ZSTD_copy8(op, match);
- }
- op += 8; match += 8;
-
- if (oMatchEnd > oend-12)
- {
- if (op < oend_8)
- {
- ZSTD_wildcopy(op, match, oend_8 - op);
- match += oend_8 - op;
- op = oend_8;
- }
- while (op < oMatchEnd) *op++ = *match++;
- }
- else
- {
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
- }
- return sequenceLength;
+ if (sequence.offset > (size_t)(oLitEnd - base))
+ {
+ /* offset beyond prefix */
+ if (sequence.offset > (size_t)(oLitEnd - vBase))
+ return ERROR(corruption_detected);
+ match = dictEnd - (base-match);
+ if (match + sequence.matchLength <= dictEnd)
+ {
+ memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ {
+ size_t length1 = dictEnd - match;
+ memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = base;
+ }
+ }
+
+ /* match within prefix */
+ if (sequence.offset < 8)
+ {
+ /* close range match, overlap */
+ const int sub2 = dec64table[sequence.offset];
+ op[0] = match[0];
+ op[1] = match[1];
+ op[2] = match[2];
+ op[3] = match[3];
+ match += dec32table[sequence.offset];
+ ZSTD_copy4(op+4, match);
+ match -= sub2;
+ }
+ else
+ {
+ ZSTD_copy8(op, match);
+ }
+ op += 8; match += 8;
+
+ if (oMatchEnd > oend-12)
+ {
+ if (op < oend_8)
+ {
+ ZSTD_wildcopy(op, match, oend_8 - op);
+ match += oend_8 - op;
+ op = oend_8;
+ }
+ while (op < oMatchEnd) *op++ = *match++;
+ }
+ else
+ {
+ ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ }
+ return sequenceLength;
}
@@ -607,9 +604,9 @@ static size_t ZSTD_decompressSequences(
U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable;
- BYTE* const base = (BYTE*) (dctx->base);
- BYTE* const vBase = (BYTE*) (dctx->vBase);
- BYTE* const dictEnd = (BYTE*) (dctx->dictEnd);
+ const BYTE* const base = (const BYTE*) (dctx->base);
+ const BYTE* const vBase = (const BYTE*) (dctx->vBase);
+ const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
/* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
@@ -634,20 +631,7 @@ static size_t ZSTD_decompressSequences(
FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
- for ( ; (BIT_reloadDStream(&(seqState.DStream)) < BIT_DStream_completed) ; )
- {
- size_t oneSeqSize;
- nbSeq--;
- ZSTD_decodeSequence(&sequence, &seqState);
- oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
- if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
- op += oneSeqSize;
- }
-
- if (nbSeq<0) return ERROR(corruption_detected); /* requested too many sequences : data is corrupted */
-
- /* now BIT_reloadDStream(&(seqState.DStream)) >= BIT_DStream_completed) */
- for ( ; (BIT_reloadDStream(&(seqState.DStream)) == BIT_DStream_completed) && nbSeq ; )
+ for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; )
{
size_t oneSeqSize;
nbSeq--;
@@ -658,7 +642,7 @@ static size_t ZSTD_decompressSequences(
}
/* check if reached exact end */
- if ( !BIT_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected); /* DStream should be entirely and precisely consumed; otherwise data is corrupted */
+ if ( !BIT_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected); /* DStream should be entirely and exactly consumed; otherwise data is corrupted */
/* last literal segment */
{
@@ -692,7 +676,10 @@ static size_t ZSTD_decompressBlock(
}
-size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
+ void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* iend = ip + srcSize;
@@ -702,9 +689,19 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v
size_t remainingSize = srcSize;
blockProperties_t blockProperties;
-
/* init */
- ctx->base = ctx->vBase = ctx->dictEnd = dst;
+ ZSTD_resetDCtx(ctx);
+ if (dict)
+ {
+ ZSTD_decompress_insertDictionary(ctx, dict, dictSize);
+ ctx->dictEnd = ctx->previousDstEnd;
+ ctx->vBase = (const char*)dst - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
+ ctx->base = dst;
+ }
+ else
+ {
+ ctx->vBase = ctx->base = ctx->dictEnd = dst;
+ }
/* Frame Header */
{
@@ -765,10 +762,16 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v
return op-ostart;
}
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+{
+ return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0);
+}
+
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
- ZSTD_DCtx ctx;
- return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
+ ZSTD_DCtx dctx;
+ return ZSTD_decompressDCtx(&dctx, dst, maxDstSize, src, srcSize);
}
@@ -786,10 +789,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
if (dst != ctx->previousDstEnd) /* not contiguous */
{
- if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment into dictionary */
- ctx->base = (char*)dst; /* temporary affectation, for vBase calculation */
ctx->dictEnd = ctx->previousDstEnd;
- ctx->vBase = (char*)dst - ((char*)(ctx->previousDstEnd) - (char*)(ctx->base));
+ ctx->vBase = (const char*)dst - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
ctx->base = dst;
ctx->previousDstEnd = dst;
}
@@ -840,10 +841,9 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
ctx->bType = bp.blockType;
ctx->stage = ZSTDds_decompressBlock;
}
-
return 0;
}
- case 3:
+ case ZSTDds_decompressBlock:
{
/* Decompress : block content */
size_t rSize;
@@ -875,3 +875,10 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
}
+void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize)
+{
+ ctx->dictEnd = ctx->previousDstEnd;
+ ctx->vBase = (const char*)src - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
+ ctx->base = src;
+ ctx->previousDstEnd = (const char*)src + srcSize;
+}
diff --git a/lib/zstd_static.h b/lib/zstd_static.h
index db31c9f..f78d464 100644
--- a/lib/zstd_static.h
+++ b/lib/zstd_static.h
@@ -80,35 +80,64 @@ typedef struct
/* *************************************
-* Advanced function
+* Advanced functions
***************************************/
/** ZSTD_getParams
* return ZSTD_parameters structure for a selected compression level and srcSize.
* srcSizeHint value is optional, select 0 if not known */
-ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint);
+ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint);
/** ZSTD_validateParams
* correct params value to remain within authorized range */
-void ZSTD_validateParams(ZSTD_parameters* params);
+ZSTDLIB_API void ZSTD_validateParams(ZSTD_parameters* params);
+
+/** ZSTD_compress_usingDict
+* Same as ZSTD_compressCCtx(), using a Dictionary content as prefix
+* Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */
+ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
+ void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ int compressionLevel);
/** ZSTD_compress_advanced
-* Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
-size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
- void* dst, size_t maxDstSize,
- const void* src, size_t srcSize,
- ZSTD_parameters params);
+* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
+ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
+ void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_parameters params);
+
+/** Decompression context management */
+typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
+ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+
+/** ZSTD_decompressDCtx
+* Same as ZSTD_decompress, with pre-allocated DCtx structure */
+size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+
+/** ZSTD_decompress_usingDict
+* Same as ZSTD_decompressDCtx, using a Dictionary content as prefix
+* Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
+ void* dst, size_t maxDstSize,
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize);
/* **************************************
-* Streaming functions (bufferless mode)
+* Streaming functions (direct mode)
****************************************/
-size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel);
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
-size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
-size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
+ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
+ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
+
+ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
/**
- Streaming compression, bufferless mode
+ Streaming compression, direct mode (bufferless)
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it.
@@ -118,6 +147,10 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
Use ZSTD_compressBegin().
You may also prefer the advanced derivative ZSTD_compressBegin_advanced(), for finer parameter control.
+ It's then possible to add a dictionary with ZSTD_compress_insertDictionary()
+ Note that dictionary presence is a "hidden" information,
+ the decoder needs to be aware that it is required for proper decoding, or decoding will fail.
+
Then, consume your input using ZSTD_compressContinue().
The interface is synchronous, so all input will be consumed.
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
@@ -125,18 +158,18 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
Without it, the frame will be considered incomplete by decoders.
- You can then re-use ZSTD_CCtx to compress new frames.
+
+ You can then reuse ZSTD_CCtx to compress new frames.
*/
-typedef struct ZSTD_DCtx_s ZSTD_DCtx;
-ZSTD_DCtx* ZSTD_createDCtx(void);
-size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
-size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
+ZSTDLIB_API void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
+
+ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
-size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
-size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
-size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/**
Streaming decompression, bufferless mode
@@ -146,15 +179,18 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
First operation is to retrieve frame parameters, using ZSTD_getFrameParams().
This function doesn't consume its input. It needs enough input data to properly decode the frame header.
- The objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding.
+ Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding.
Result : 0 when successful, it means the ZSTD_parameters structure has been filled.
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header)
+ Then, you can optionally insert a dictionary.
+ This operation must mimic the compressor behavior, otherwise decompression will fail or be corrupted.
+
Then it's possible to start decompression.
Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
- ZSTD_decompressContinue() requires this exact amount of bytes, or just fails.
+ ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
@@ -169,9 +205,10 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
* Pre-defined compression levels
***************************************/
#define ZSTD_MAX_CLEVEL 20
-static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
+ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
+static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = {
{ /* "default" */
- /* W, C, H, S, L, strat */
+ /* W, C, H, S, L, strat */
{ 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
{ 0, 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */
{ 0, 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */
@@ -194,8 +231,32 @@ static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
{ 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
{ 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */
},
+{ /* for srcSize <= 256 KB */
+ /* W, C, H, S, L, strat */
+ { 0, 18, 13, 14, 1, 7, ZSTD_fast }, /* level 0 - never used */
+ { 0, 18, 14, 15, 1, 6, ZSTD_fast }, /* level 1 */
+ { 0, 18, 14, 15, 1, 5, ZSTD_fast }, /* level 2 */
+ { 0, 18, 12, 15, 3, 7, ZSTD_greedy }, /* level 3 */
+ { 0, 18, 13, 15, 4, 7, ZSTD_greedy }, /* level 4 */
+ { 0, 18, 14, 15, 5, 7, ZSTD_greedy }, /* level 5 */
+ { 0, 18, 13, 15, 4, 7, ZSTD_lazy }, /* level 6 */
+ { 0, 18, 14, 16, 5, 7, ZSTD_lazy }, /* level 7 */
+ { 0, 18, 15, 16, 6, 7, ZSTD_lazy }, /* level 8 */
+ { 0, 18, 15, 15, 7, 7, ZSTD_lazy }, /* level 9 */
+ { 0, 18, 16, 16, 7, 7, ZSTD_lazy }, /* level 10 */
+ { 0, 18, 16, 16, 8, 4, ZSTD_lazy }, /* level 11 */
+ { 0, 18, 17, 16, 8, 4, ZSTD_lazy }, /* level 12 */
+ { 0, 18, 17, 16, 9, 4, ZSTD_lazy }, /* level 13 */
+ { 0, 18, 18, 16, 9, 4, ZSTD_lazy }, /* level 14 */
+ { 0, 18, 17, 17, 9, 4, ZSTD_lazy2 }, /* level 15 */
+ { 0, 18, 18, 18, 9, 4, ZSTD_lazy2 }, /* level 16 */
+ { 0, 18, 18, 18, 10, 4, ZSTD_lazy2 }, /* level 17 */
+ { 0, 18, 18, 18, 11, 4, ZSTD_lazy2 }, /* level 18 */
+ { 0, 18, 18, 18, 12, 4, ZSTD_lazy2 }, /* level 19 */
+ { 0, 18, 18, 18, 13, 4, ZSTD_lazy2 }, /* level 20 */
+},
{ /* for srcSize <= 128 KB */
- /* W, C, H, S, L, strat */
+ /* W, C, H, S, L, strat */
{ 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
{ 0, 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */
{ 0, 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */
@@ -218,6 +279,30 @@ static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
{ 0, 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */
{ 0, 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */
},
+{ /* for srcSize <= 16 KB */
+ /* W, C, H, S, L, strat */
+ { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */
+ { 0, 14, 14, 14, 1, 4, ZSTD_fast }, /* level 1 */
+ { 0, 14, 14, 16, 1, 4, ZSTD_fast }, /* level 1 */
+ { 0, 14, 14, 14, 5, 4, ZSTD_greedy }, /* level 3 */
+ { 0, 14, 14, 14, 8, 4, ZSTD_greedy }, /* level 4 */
+ { 0, 14, 11, 14, 6, 4, ZSTD_lazy }, /* level 5 */
+ { 0, 14, 14, 13, 6, 5, ZSTD_lazy }, /* level 6 */
+ { 0, 14, 14, 14, 7, 6, ZSTD_lazy }, /* level 7 */
+ { 0, 14, 14, 14, 8, 4, ZSTD_lazy }, /* level 8 */
+ { 0, 14, 14, 15, 9, 4, ZSTD_lazy }, /* level 9 */
+ { 0, 14, 14, 15, 10, 4, ZSTD_lazy }, /* level 10 */
+ { 0, 14, 15, 15, 6, 4, ZSTD_btlazy2 }, /* level 11 */
+ { 0, 14, 15, 15, 7, 4, ZSTD_btlazy2 }, /* level 12 */
+ { 0, 14, 15, 15, 8, 4, ZSTD_btlazy2 }, /* level 13 */
+ { 0, 14, 15, 15, 9, 4, ZSTD_btlazy2 }, /* level 14 */
+ { 0, 14, 15, 15, 10, 4, ZSTD_btlazy2 }, /* level 15 */
+ { 0, 14, 15, 15, 11, 4, ZSTD_btlazy2 }, /* level 16 */
+ { 0, 14, 15, 15, 12, 4, ZSTD_btlazy2 }, /* level 17 */
+ { 0, 14, 15, 15, 13, 4, ZSTD_btlazy2 }, /* level 18 */
+ { 0, 14, 15, 15, 14, 4, ZSTD_btlazy2 }, /* level 19 */
+ { 0, 14, 15, 15, 15, 4, ZSTD_btlazy2 }, /* level 20 */
+},
};
diff --git a/programs/Makefile b/programs/Makefile
index 9d73cb9..57fa87c 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
-VERSION?= 0.4.0
+VERSION?= 0.4.5
DESTDIR?=
PREFIX ?= /usr/local
@@ -46,7 +46,7 @@ ZSTDDIR = ../lib
ZSTD_FILES := $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c
ZSTD_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c
-ifeq ($(ZSTD_LEGACY),disable)
+ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
else
ZSTD_FILES+= $(ZSTD_LEGACY)
@@ -65,6 +65,7 @@ VOID = /dev/null
endif
ZBUFFTEST = -T2mn
+FUZZERTEST= -T5mn
.PHONY: default all clean install uninstall test test32 test-all
@@ -73,13 +74,29 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen
zstd : $(ZSTD_FILES) $(ZSTDDIR)/zstd_buffered.c \
- xxhash.c bench.c fileio.c zstdcli.c $(ZSTD_FILEIO_LEGACY)
+ zstdcli.c fileio.c $(ZSTD_FILEIO_LEGACY) bench.c xxhash.c datagen.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTD_FILES) $(ZSTDDIR)/zstd_buffered.c \
- xxhash.c bench.c fileio.c zstdcli.c $(ZSTD_FILEIO_LEGACY)
+ zstdcli.c fileio.c $(ZSTD_FILEIO_LEGACY) bench.c xxhash.c datagen.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+zstd_nolegacy :
+ $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
+
+zstd-pgo : MOREFLAGS = -fprofile-generate
+zstd-pgo : clean zstd
+ ./zstd -b $(PROFILE_WITH)
+ rm zstd
+ $(MAKE) zstd MOREFLAGS=-fprofile-use
+
+zstd-noBench: $(ZSTD_FILES) $(ZSTDDIR)/zstd_buffered.c \
+ zstdcli.c fileio.c $(ZSTD_FILEIO_LEGACY)
+ $(CC) $(FLAGS) -DZSTD_NOBENCH $^ -o zstd$(EXT)
+
+zstd-frugal: clean
+ $(MAKE) zstd-noBench ZSTD_LEGACY_SUPPORT=0
+
fullbench : $(ZSTD_FILES) \
datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -112,7 +129,7 @@ datagen : datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean:
- @rm -f core *.o tmp* \
+ @rm -f core *.o tmp* result* *.gcda \
zstd$(EXT) zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
@@ -152,77 +169,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
test-all: test test32 valgrindTest
zstd-playTests: datagen
- @echo "\n**** frame concatenation **** "
- @echo "hello " > hello.tmp
- @echo "world!" > world.tmp
- @cat hello.tmp world.tmp > helloworld.tmp
- $(ZSTD) hello.tmp > hello.zstd
- $(ZSTD) world.tmp > world.zstd
- @cat hello.zstd world.zstd > helloworld.zstd
- $(ZSTD) -df helloworld.zstd > result.tmp
- cat result.tmp
- sdiff helloworld.tmp result.tmp
- @rm *.tmp *.zstd
- @echo frame concatenation test completed
- @echo "**** flush write error test **** "
- echo foo | $(ZSTD) > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
- echo foo | $(ZSTD) | $(ZSTD) -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
- @echo "**** zstd round-trip tests **** "
- @./datagen | md5sum > tmp1
- ./datagen | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen | $(ZSTD) -6 -v | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- @./datagen -g270000000 | md5sum > tmp1
- ./datagen -g270000000 | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g270000000 | $(ZSTD) -v2 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g270000000 | $(ZSTD) -v3 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- @./datagen -g140000000 -P60| md5sum > tmp1
- ./datagen -g140000000 -P60 | $(ZSTD) -v4 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g140000000 -P60 | $(ZSTD) -v5 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g140000000 -P60 | $(ZSTD) -v6 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- @./datagen -g70000000 -P70 | md5sum > tmp1
- ./datagen -g70000000 -P70 | $(ZSTD) -v7 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g70000000 -P70 | $(ZSTD) -v8 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g70000000 -P70 | $(ZSTD) -v9 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- @./datagen -g35000000 -P75 | md5sum > tmp1
- ./datagen -g35000000 -P75 | $(ZSTD) -v10 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g35000000 -P75 | $(ZSTD) -v11 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g35000000 -P75 | $(ZSTD) -v12 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- @./datagen -g18000000 -P80 | md5sum > tmp1
- ./datagen -g18000000 -P80 | $(ZSTD) -v13 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g18000000 -P80 | $(ZSTD) -v14 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g18000000 -P80 | $(ZSTD) -v15 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g18000000 -P80 | $(ZSTD) -v16 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g18000000 -P80 | $(ZSTD) -v17 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- @./datagen -g50000000 -P94 | md5sum > tmp1
- ./datagen -g50000000 -P94 | $(ZSTD) -v18 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g50000000 -P94 | $(ZSTD) -v19 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- @./datagen -g99000000 -P99 | md5sum > tmp1
- ./datagen -g99000000 -P99 | $(ZSTD) -v20 | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
- ./datagen -g6000000000 -P99| md5sum > tmp1
- ./datagen -g6000000000 -P99| $(ZSTD) -vq | $(ZSTD) -d | md5sum > tmp2
- @diff tmp1 tmp2
+ ZSTD=$(ZSTD) ./playTests.sh --test-large-data
test-zstd: ZSTD = ./zstd
test-zstd: zstd zstd-playTests
@@ -230,6 +177,9 @@ test-zstd: zstd zstd-playTests
test-zstd32: ZSTD = ./zstd32
test-zstd32: zstd32 zstd-playTests
+test-zstd_nolegacy: ZSTD = ./zstd
+test-zstd_nolegacy: zstd_nolegacy zstd-playTests
+
test-fullbench: fullbench datagen
./fullbench -i1
./fullbench -i1 -P0
@@ -239,10 +189,10 @@ test-fullbench32: fullbench32 datagen
./fullbench32 -i1 -P0
test-fuzzer: fuzzer
- ./fuzzer
+ ./fuzzer $(FUZZERTEST)
test-fuzzer32: fuzzer32
- ./fuzzer32
+ ./fuzzer32 $(FUZZERTEST)
test-zbuff: zbufftest
./zbufftest $(ZBUFFTEST)
@@ -261,7 +211,7 @@ valgrindTest: zstd datagen fuzzer fullbench zbufftest
./datagen -g64MB > tmp
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
@rm tmp
- valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1
+ valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -T1mn -t1
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
valgrind --leak-check=yes --error-exitcode=1 ./zbufftest -T1mn
diff --git a/programs/bench.c b/programs/bench.c
index fa7ec55..b80d699 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -27,7 +27,10 @@
* Compiler Options
****************************************/
/* Disable some Visual warning messages */
-#define _CRT_SECURE_NO_WARNINGS /* fopen */
+#ifdef _MSC_VER
+# define _CRT_SECURE_NO_WARNINGS /* fopen */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
/* Unix Large Files support (>4GB) */
#define _FILE_OFFSET_BITS 64
@@ -54,14 +57,15 @@
/* Use ftime() if gettimeofday() is not available */
#if defined(BMK_LEGACY_TIMER)
-# include <sys/timeb.h> /* timeb, ftime */
+# include <sys/timeb.h> /* timeb, ftime */
#else
-# include <sys/time.h> /* gettimeofday */
+# include <sys/time.h> /* gettimeofday */
#endif
#include "mem.h"
-#include "zstd.h"
+#include "zstd_static.h"
#include "xxhash.h"
+#include "datagen.h" /* RDG_genBuffer */
/* *************************************
@@ -71,6 +75,10 @@
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
+#ifdef _MSC_VER
+#define snprintf sprintf_s
+#endif
+
/* *************************************
* Constants
@@ -82,18 +90,35 @@
#define MB *(1 <<20)
#define GB *(1U<<30)
-static const size_t maxMemory = sizeof(size_t)==4 ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
+static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
#define DEFAULT_CHUNKSIZE (4 MB)
static U32 g_compressibilityDefault = 50;
-static U32 prime1 = 2654435761U;
-static U32 prime2 = 2246822519U;
/* *************************************
-* Macros
+* console display
+***************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
+
+
+/* *************************************
+* Exceptions
***************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, "\n"); \
+ exit(error); \
+}
/* *************************************
@@ -157,60 +182,18 @@ static int BMK_GetMilliSpan( int nTimeStart )
return nSpan;
}
-
-/* ********************************************************
-* Data generator
-**********************************************************/
-/* will hopefully be converted into ROL instruction by compiler */
-static U32 BMK_rotl32(unsigned val32, unsigned nbBits) { return((val32 << nbBits) | (val32 >> (32 - nbBits))); }
-
-static U32 BMK_rand(U32* src)
-{
- U32 rand32 = *src;
- rand32 *= prime1;
- rand32 += prime2;
- rand32 = BMK_rotl32(rand32, 13);
- *src = rand32;
- return rand32 >> 9;
-}
-
-#define BMK_RAND15BITS ( BMK_rand(&seed) & 0x7FFF)
-#define BMK_RANDLENGTH ((BMK_rand(&seed) & 3) ? (BMK_rand(&seed) % 15) : (BMK_rand(&seed) % 510) + 15)
-#define BMK_RANDCHAR (BYTE)((BMK_rand(&seed) & 63) + '0')
-static void BMK_datagen(void* buffer, size_t bufferSize, double proba, U32 seed)
+static U64 BMK_getFileSize(const char* infilename)
{
- BYTE* BBuffer = (BYTE*)buffer;
- unsigned pos = 0;
- U32 P32 = (U32)(32768 * proba);
-
- /* First Byte */
- BBuffer[pos++] = BMK_RANDCHAR;
-
- while (pos < bufferSize)
- {
- /* Select : Literal (noise) or copy (within 64K) */
- if (BMK_RAND15BITS < P32)
- {
- /* Match */
- size_t match, end;
- unsigned length = BMK_RANDLENGTH + 4;
- unsigned offset = BMK_RAND15BITS + 1;
- if (offset > pos) offset = pos;
- match = pos - offset;
- end = pos + length;
- if (end > bufferSize) end = bufferSize;
- while (pos < end) BBuffer[pos++] = BBuffer[match++];
- }
- else
- {
- /* Literal */
- size_t end;
- unsigned length = BMK_RANDLENGTH;
- end = pos + length;
- if (end > bufferSize) end = bufferSize;
- while (pos < end) BBuffer[pos++] = BMK_RANDCHAR;
- }
- }
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+#endif
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+ return (U64)statbuf.st_size;
}
@@ -219,7 +202,7 @@ static void BMK_datagen(void* buffer, size_t bufferSize, double proba, U32 seed)
**********************************************************/
typedef struct
{
- char* srcPtr;
+ const char* srcPtr;
size_t srcSize;
char* cPtr;
size_t cRoom;
@@ -228,62 +211,60 @@ typedef struct
size_t resSize;
} blockParam_t;
-typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
-
#define MIN(a,b) ((a)<(b) ? (a) : (b))
-static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
+static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
+ const char* displayName, int cLevel,
+ const size_t* fileSizes, U32 nbFiles,
+ const void* dictBuffer, size_t dictBufferSize)
{
const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
- const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
- blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t));
- const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
+ const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+ blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
+ const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
- const compressor_t compressor = ZSTD_compress;
- U64 crcOrig;
+ ZSTD_CCtx* ctx = ZSTD_createCCtx();
+ ZSTD_DCtx* dctx = ZSTD_createDCtx();
+ U64 crcOrig = XXH64(srcBuffer, srcSize, 0);
+ U32 nbBlocks = 0;
/* init */
- if (strlen(fileName)>16)
- fileName += strlen(fileName)-16;
+ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
/* Memory allocation & restrictions */
- if (!compressedBuffer || !resultBuffer || !blockTable)
- {
- DISPLAY("\nError: not enough memory!\n");
- free(compressedBuffer);
- free(resultBuffer);
- free(blockTable);
- return 12;
- }
-
- /* Calculating input Checksum */
- crcOrig = XXH64(srcBuffer, srcSize, 0);
+ if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
+ EXM_THROW(31, "not enough memory");
/* Init blockTable data */
{
- U32 i;
- size_t remaining = srcSize;
- char* srcPtr = (char*)srcBuffer;
+ U32 fileNb;
+ const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
- for (i=0; i<nbBlocks; i++)
+ for (fileNb=0; fileNb<nbFiles; fileNb++)
{
- size_t thisBlockSize = MIN(remaining, blockSize);
- blockTable[i].srcPtr = srcPtr;
- blockTable[i].cPtr = cPtr;
- blockTable[i].resPtr = resPtr;
- blockTable[i].srcSize = thisBlockSize;
- blockTable[i].cRoom = ZSTD_compressBound(thisBlockSize);
- srcPtr += thisBlockSize;
- cPtr += blockTable[i].cRoom;
- resPtr += thisBlockSize;
- remaining -= thisBlockSize;
+ size_t remaining = fileSizes[fileNb];
+ U32 nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
+ U32 blockEnd = nbBlocks + nbBlocksforThisFile;
+ for ( ; nbBlocks<blockEnd; nbBlocks++)
+ {
+ size_t thisBlockSize = MIN(remaining, blockSize);
+ blockTable[nbBlocks].srcPtr = srcPtr;
+ blockTable[nbBlocks].cPtr = cPtr;
+ blockTable[nbBlocks].resPtr = resPtr;
+ blockTable[nbBlocks].srcSize = thisBlockSize;
+ blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
+ srcPtr += thisBlockSize;
+ cPtr += blockTable[nbBlocks].cRoom;
+ resPtr += thisBlockSize;
+ remaining -= thisBlockSize;
+ }
}
}
/* warmimg up memory */
- BMK_datagen(compressedBuffer, maxCompressedSize, 0.10, 1);
+ RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
/* Bench */
{
@@ -301,7 +282,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
U32 blockNb;
/* Compression */
- DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, fileName, (U32)srcSize);
+ DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize);
memset(compressedBuffer, 0xE5, maxCompressedSize);
nbLoops = 0;
@@ -311,7 +292,11 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (blockNb=0; blockNb<nbBlocks; blockNb++)
- blockTable[blockNb].cSize = compressor(blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
+ blockTable[blockNb].cSize = ZSTD_compress_usingDict(ctx,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
+ blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
+ dictBuffer, dictBufferSize,
+ cLevel);
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
@@ -321,7 +306,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
ratio = (double)srcSize / (double)cSize;
- DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
+ DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
#if 1
/* Decompression */
@@ -334,13 +319,15 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++)
{
for (blockNb=0; blockNb<nbBlocks; blockNb++)
- blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
- blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
+ blockTable[blockNb].resSize = ZSTD_decompress_usingDict(dctx,
+ blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
+ dictBuffer, dictBufferSize);
}
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
- DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+ DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
/* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -348,10 +335,10 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
{
unsigned u;
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
- DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
+ DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++)
{
- if (((BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u])
+ if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u])
{
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
@@ -363,30 +350,20 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
}
if (crcOrig == crcCheck)
- DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+ DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+ else
+ DISPLAY("X \n");
}
- /* End cleaning */
+ /* clean up */
free(compressedBuffer);
free(resultBuffer);
+ ZSTD_freeCCtx(ctx);
+ ZSTD_freeDCtx(dctx);
return 0;
}
-static U64 BMK_GetFileSize(const char* infilename)
-{
- int r;
-#if defined(_MSC_VER)
- struct _stat64 statbuf;
- r = _stat64(infilename, &statbuf);
-#else
- struct stat statbuf;
- r = stat(infilename, &statbuf);
-#endif
- if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
- return (U64)statbuf.st_size;
-}
-
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t step = 64 MB;
@@ -406,124 +383,143 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem - step);
}
-static int BMK_benchOneFile(const char* inFileName, int cLevel)
+static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+ const char* displayName, int cLevel,
+ const size_t* fileSizes, unsigned nbFiles,
+ const void* dictBuffer, size_t dictBufferSize)
{
- FILE* inFile;
- U64 inFileSize;
- size_t benchedSize, readSize;
- void* srcBuffer;
- int result=0;
-
- /* Check file existence */
- inFile = fopen(inFileName, "rb");
- if (inFile == NULL)
+ if (cLevel < 0)
{
- DISPLAY("Pb opening %s\n", inFileName);
- return 11;
+ int l;
+ for (l=1; l <= -cLevel; l++)
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize);
+ return;
}
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, cLevel,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize);
+}
- /* Memory allocation & restrictions */
- inFileSize = BMK_GetFileSize(inFileName);
- benchedSize = BMK_findMaxMem(inFileSize * 3) / 3;
- if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
- if (benchedSize < inFileSize)
- DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize >> 20));
- srcBuffer = malloc(benchedSize);
- if (!srcBuffer)
- {
- DISPLAY("\nError: not enough memory!\n");
- fclose(inFile);
- return 12;
- }
+static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+ U64 total = 0;
+ unsigned n;
+ for (n=0; n<nbFiles; n++)
+ total += BMK_getFileSize(fileNamesTable[n]);
+ return total;
+}
- /* Fill input buffer */
- DISPLAY("Loading %s... \r", inFileName);
- readSize = fread(srcBuffer, 1, benchedSize, inFile);
- fclose(inFile);
+static void BMK_loadFiles(void* buffer, size_t bufferSize,
+ size_t* fileSizes,
+ const char** fileNamesTable, unsigned nbFiles)
+{
+ BYTE* buff = (BYTE*)buffer;
+ size_t pos = 0;
+ unsigned n;
- if (readSize != benchedSize)
+ for (n=0; n<nbFiles; n++)
{
- DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
- free(srcBuffer);
- return 13;
+ size_t readSize;
+ U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
+ FILE* f = fopen(fileNamesTable[n], "rb");
+ if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+ DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
+ if (fileSize > bufferSize-pos) fileSize = bufferSize-pos;
+ readSize = fread(buff+pos, 1, (size_t)fileSize, f);
+ if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+ pos += readSize;
+ fileSizes[n] = (size_t)fileSize;
+ fclose(f);
}
+}
- /* Bench */
- if (cLevel<0)
+static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
+ const char* dictFileName, int cLevel)
+{
+ void* srcBuffer;
+ size_t benchedSize;
+ void* dictBuffer = NULL;
+ size_t dictBufferSize = 0;
+ size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+ U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles);
+ char mfName[20] = {0};
+ const char* displayName = NULL;
+
+ if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+
+ /* Load dictionary */
+ if (dictFileName != NULL)
{
- int l;
- for (l=1; l <= -cLevel; l++)
- result = BMK_benchMem(srcBuffer, benchedSize, inFileName, l);
+ U64 dictFileSize = BMK_getFileSize(dictFileName);
+ if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
+ dictBufferSize = (size_t)dictFileSize;
+ dictBuffer = malloc(dictBufferSize);
+ if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
+ BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
}
- else
- result = BMK_benchMem(srcBuffer, benchedSize, inFileName, cLevel);
+
+ /* Memory allocation & restrictions */
+ benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
+ if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
+ if (benchedSize < totalSizeToLoad)
+ DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+ srcBuffer = malloc(benchedSize);
+ if (!srcBuffer) EXM_THROW(12, "not enough memory");
+
+ /* Load input buffer */
+ BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
+
+ /* Bench */
+ snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+ if (nbFiles > 1) displayName = mfName;
+ else displayName = fileNamesTable[0];
+
+ BMK_benchCLevel(srcBuffer, benchedSize,
+ displayName, cLevel,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize);
/* clean up */
free(srcBuffer);
- DISPLAY("\n");
- return result;
+ free(dictBuffer);
+ free(fileSizes);
}
-static int BMK_syntheticTest(int cLevel, double compressibility)
+static void BMK_syntheticTest(int cLevel, double compressibility)
{
+ char name[20] = {0};
size_t benchedSize = 10000000;
void* srcBuffer = malloc(benchedSize);
- int result=0;
- char name[20] = {0};
/* Memory allocation */
- if (!srcBuffer)
- {
- DISPLAY("\nError: not enough memory!\n");
- free(srcBuffer);
- return 12;
- }
+ if (!srcBuffer) EXM_THROW(21, "not enough memory");
/* Fill input buffer */
- BMK_datagen(srcBuffer, benchedSize, compressibility, 0);
+ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */
-#ifdef _MSC_VER
- sprintf_s(name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100));
-#else
- snprintf (name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100));
-#endif
- /* Bench */
- if (cLevel<0)
- {
- int l;
- for (l=1; l <= -cLevel; l++)
- result = BMK_benchMem(srcBuffer, benchedSize, name, l);
- }
- else
- result = BMK_benchMem(srcBuffer, benchedSize, name, cLevel);
+ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
+ BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0);
- /* End */
+ /* clean up */
free(srcBuffer);
- DISPLAY("\n");
- return result;
}
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+ const char* dictFileName, int cLevel)
{
double compressibility = (double)g_compressibilityDefault / 100;
if (nbFiles == 0)
- {
BMK_syntheticTest(cLevel, compressibility);
- }
else
- {
- /* Loop for each file */
- unsigned fileIdx = 0;
- while (fileIdx<nbFiles)
- {
- BMK_benchOneFile(fileNamesTable[fileIdx], cLevel);
- fileIdx++;
- }
- }
+ BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel);
return 0;
}
diff --git a/programs/bench.h b/programs/bench.h
index 3cc67b4..9ae8369 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -26,7 +26,8 @@
/* Main function */
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+ const char* dictFileName, int cLevel);
/* Set Parameters */
void BMK_SetNbIterations(int nbLoops);
diff --git a/programs/datagen.c b/programs/datagen.c
index f5aa9c5..2bb3426 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -173,6 +173,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
U32 repeatOffset = (RDG_rand(seed) & 15) == 2;
if (repeatOffset) offset = prevOffset;
if (offset > pos) offset = (U32)pos;
+ prevOffset = offset;
match = pos - offset;
d = pos + length;
if (d > buffSize) d = buffSize;
diff --git a/programs/datagen.h b/programs/datagen.h
index 03b06ca..de4a074 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -28,7 +28,7 @@
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed);
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed);
-/* RDG_genBuffer
+/*!RDG_genBuffer
Generate 'size' bytes of compressible data into 'buffer'.
Compressibility can be controlled using 'matchProba', which is floating point value between 0 and 1.
'LitProba' is optional, it affect variability of individual bytes. If litProba==0.0, default value will be used.
diff --git a/programs/fileio.c b/programs/fileio.c
index e84d633..3867301 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -121,6 +121,7 @@
#define CACHELINE 64
+#define MAX_DICT_SIZE (512 KB)
/* *************************************
* Macros
@@ -174,100 +175,197 @@ static unsigned FIO_GetMilliSpan(clock_t nPrevious)
}
-static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* input_filename, const char* output_filename)
+static U64 FIO_getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+#endif
+ if (r || !S_ISREG(statbuf.st_mode)) return 0;
+ return (U64)statbuf.st_size;
+}
+
+
+static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
+ const char* dstFileName, const char* srcFileName)
{
- if (!strcmp (input_filename, stdinmark))
+ if (!strcmp (srcFileName, stdinmark))
{
DISPLAYLEVEL(4,"Using stdin for input\n");
- *pfinput = stdin;
+ *fileInPtr = stdin;
SET_BINARY_MODE(stdin);
}
else
{
- *pfinput = fopen(input_filename, "rb");
+ *fileInPtr = fopen(srcFileName, "rb");
+ }
+
+ if ( *fileInPtr==0 )
+ {
+ DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
+ return 1;
}
- if (!strcmp (output_filename, stdoutmark))
+ if (!strcmp (dstFileName, stdoutmark))
{
DISPLAYLEVEL(4,"Using stdout for output\n");
- *pfoutput = stdout;
+ *fileOutPtr = stdout;
SET_BINARY_MODE(stdout);
}
else
{
/* Check if destination file already exists */
- *pfoutput=0;
- if (strcmp(output_filename,nulmark)) *pfoutput = fopen( output_filename, "rb" );
- if (*pfoutput!=0)
+ if (!g_overwrite)
{
- fclose(*pfoutput);
- if (!g_overwrite)
+ *fileOutPtr = fopen( dstFileName, "rb" );
+ if (*fileOutPtr != 0)
{
- char ch;
- if (g_displayLevel <= 1) /* No interaction possible */
- EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
- DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
- DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
- ch = (char)getchar();
- if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
+ /* prompt for overwrite authorization */
+ int ch = 'N';
+ fclose(*fileOutPtr);
+ DISPLAY("Warning : %s already exists \n", dstFileName);
+ if ((g_displayLevel <= 1) || (*fileInPtr == stdin))
+ {
+ /* No interaction possible */
+ DISPLAY("Operation aborted : %s already exists \n", dstFileName);
+ return 1;
+ }
+ DISPLAY("Overwrite ? (y/N) : ");
+ while((ch = getchar()) != '\n' && ch != EOF); /* flush integrated */
+ if ((ch!='Y') && (ch!='y'))
+ {
+ DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
+ return 1;
+ }
}
}
- *pfoutput = fopen( output_filename, "wb" );
+ *fileOutPtr = fopen( dstFileName, "wb" );
}
- if ( *pfinput==0 ) EXM_THROW(12, "Pb opening src : %s", input_filename);
- if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
+ if (*fileOutPtr==0) EXM_THROW(13, "Pb opening %s", dstFileName);
+
+ return 0;
+}
+
+/*!FIO_loadFile
+* creates a buffer, pointed by *bufferPtr,
+* loads "filename" content into it
+* up to MAX_DICT_SIZE bytes
+*/
+static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
+{
+ FILE* fileHandle;
+ size_t readSize;
+ U64 fileSize;
+
+ *bufferPtr = NULL;
+ if (fileName == NULL)
+ return 0;
+
+ DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
+ fileHandle = fopen(fileName, "rb");
+ if (fileHandle==0) EXM_THROW(31, "Error opening file %s", fileName);
+ fileSize = FIO_getFileSize(fileName);
+ if (fileSize > MAX_DICT_SIZE)
+ {
+ int seekResult;
+ if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName); /* avoid extreme cases */
+ DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, MAX_DICT_SIZE);
+ seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET); /* use end of file */
+ if (seekResult != 0) EXM_THROW(33, "Error seeking into file %s", fileName);
+ fileSize = MAX_DICT_SIZE;
+ }
+ *bufferPtr = (BYTE*)malloc((size_t)fileSize);
+ if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
+ readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
+ if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName);
+ fclose(fileHandle);
+ return (size_t)fileSize;
}
-static U64 FIO_getFileSize(const char* infilename)
+/* **********************************************************************
+* Compression
+************************************************************************/
+typedef struct {
+ void* srcBuffer;
+ size_t srcBufferSize;
+ void* dstBuffer;
+ size_t dstBufferSize;
+ void* dictBuffer;
+ size_t dictBufferSize;
+ ZBUFF_CCtx* ctx;
+} cRess_t;
+
+static cRess_t FIO_createCResources(const char* dictFileName)
{
- int r;
-#if defined(_MSC_VER)
- struct _stat64 statbuf;
- r = _stat64(infilename, &statbuf);
-#else
- struct stat statbuf;
- r = stat(infilename, &statbuf);
-#endif
- if (r || !S_ISREG(statbuf.st_mode)) return 0;
- return (U64)statbuf.st_size;
+ cRess_t ress;
+
+ ress.ctx = ZBUFF_createCCtx();
+ if (ress.ctx == NULL) EXM_THROW(30, "Allocation error : can't create ZBUFF context");
+
+ /* Allocate Memory */
+ ress.srcBufferSize = ZBUFF_recommendedCInSize();
+ ress.srcBuffer = malloc(ress.srcBufferSize);
+ ress.dstBufferSize = ZBUFF_recommendedCOutSize();
+ ress.dstBuffer = malloc(ress.dstBufferSize);
+ if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
+
+ /* dictionary */
+ ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
+
+ return ress;
+}
+
+static void FIO_freeCResources(cRess_t ress)
+{
+ size_t errorCode;
+ free(ress.srcBuffer);
+ free(ress.dstBuffer);
+ free(ress.dictBuffer);
+ errorCode = ZBUFF_freeCCtx(ress.ctx);
+ if (ZBUFF_isError(errorCode)) EXM_THROW(38, "Error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
}
-unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel)
+/*
+ * FIO_compressFilename_extRess()
+ * result : 0 : compression completed correctly
+ * 1 : missing or pb opening srcFileName
+ */
+static int FIO_compressFilename_extRess(cRess_t ress,
+ const char* dstFileName, const char* srcFileName,
+ int cLevel)
{
+ FILE* srcFile;
+ FILE* dstFile;
U64 filesize = 0;
U64 compressedfilesize = 0;
- BYTE* inBuff;
- BYTE* outBuff;
- size_t inBuffSize = ZBUFF_recommendedCInSize();
- size_t outBuffSize = ZBUFF_recommendedCOutSize();
- FILE* finput;
- FILE* foutput;
+ size_t dictSize = ress.dictBufferSize;
size_t sizeCheck, errorCode;
- ZBUFF_CCtx* ctx;
- /* Allocate Memory */
- ctx = ZBUFF_createCCtx();
- inBuff = (BYTE*)malloc(inBuffSize);
- outBuff = (BYTE*)malloc(outBuffSize);
- if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
+ /* File check */
+ if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
/* init */
- FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
- filesize = FIO_getFileSize(input_filename);
- errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize));
- if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression");
- filesize = 0;
+ filesize = FIO_getFileSize(srcFileName) + dictSize;
+ errorCode = ZBUFF_compressInit_advanced(ress.ctx, ZSTD_getParams(cLevel, filesize));
+ if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression");
+ errorCode = ZBUFF_compressWithDictionary(ress.ctx, ress.dictBuffer, ress.dictBufferSize);
+ if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing dictionary");
/* Main compression loop */
+ filesize = 0;
while (1)
{
size_t inSize;
/* Fill input Buffer */
- inSize = fread(inBuff, (size_t)1, inBuffSize, finput);
+ inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
if (inSize==0) break;
filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
@@ -275,8 +373,8 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
{
/* Compress (buffered streaming ensures appropriate formatting) */
size_t usedInSize = inSize;
- size_t cSize = outBuffSize;
- size_t result = ZBUFF_compressContinue(ctx, outBuff, &cSize, inBuff, &usedInSize);
+ size_t cSize = ress.dstBufferSize;
+ size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
if (ZBUFF_isError(result))
EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
if (inSize != usedInSize)
@@ -284,8 +382,8 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
EXM_THROW(24, "Compression error : input block not fully consumed");
/* Write cBlock */
- sizeCheck = fwrite(outBuff, 1, cSize, foutput);
- if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
+ sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
+ if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
compressedfilesize += cSize;
}
@@ -294,12 +392,12 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* End of Frame */
{
- size_t cSize = outBuffSize;
- size_t result = ZBUFF_compressEnd(ctx, outBuff, &cSize);
+ size_t cSize = ress.dstBufferSize;
+ size_t result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
- sizeCheck = fwrite(outBuff, 1, cSize, foutput);
- if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
+ sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
+ if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);
compressedfilesize += cSize;
}
@@ -309,49 +407,149 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
/* clean */
- free(inBuff);
- free(outBuff);
- ZBUFF_freeCCtx(ctx);
- fclose(finput);
- if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
+ fclose(srcFile);
+ if (fclose(dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
- return compressedfilesize;
+ return 0;
}
-unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
- BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded,
- BYTE* outBuff, size_t outBuffSize,
- ZBUFF_DCtx* dctx)
+int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
+ const char* dictFileName, int compressionLevel)
+{
+ clock_t start, end;
+ cRess_t ress;
+ int issueWithSrcFile = 0;
+
+ /* Init */
+ start = clock();
+ ress = FIO_createCResources(dictFileName);
+
+ /* Compress File */
+ issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
+
+ /* Free resources */
+ FIO_freeCResources(ress);
+
+ /* Final Status */
+ end = clock();
+ {
+ double seconds = (double)(end - start) / CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
+ }
+
+ return issueWithSrcFile;
+}
+
+
+#define FNSPACE 30
+int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
+ const char* suffix,
+ const char* dictFileName, int compressionLevel)
+{
+ unsigned u;
+ int missed_files = 0;
+ char* dstFileName = (char*)malloc(FNSPACE);
+ size_t dfnSize = FNSPACE;
+ const size_t suffixSize = strlen(suffix);
+ cRess_t ress;
+
+ /* init */
+ ress = FIO_createCResources(dictFileName);
+
+ /* loop on each file */
+ for (u=0; u<nbFiles; u++)
+ {
+ size_t ifnSize = strlen(inFileNamesTable[u]);
+ if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
+ strcpy(dstFileName, inFileNamesTable[u]);
+ strcat(dstFileName, suffix);
+
+ missed_files += FIO_compressFilename_extRess(ress, dstFileName, inFileNamesTable[u], compressionLevel);
+ }
+
+ /* Close & Free */
+ FIO_freeCResources(ress);
+ free(dstFileName);
+
+ return missed_files;
+}
+
+
+/* **************************************************************************
+* Decompression
+****************************************************************************/
+typedef struct {
+ void* srcBuffer;
+ size_t srcBufferSize;
+ void* dstBuffer;
+ size_t dstBufferSize;
+ void* dictBuffer;
+ size_t dictBufferSize;
+ ZBUFF_DCtx* dctx;
+} dRess_t;
+
+static dRess_t FIO_createDResources(const char* dictFileName)
+{
+ dRess_t ress;
+
+ /* init */
+ ress.dctx = ZBUFF_createDCtx();
+ if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
+
+ /* Allocate Memory */
+ ress.srcBufferSize = ZBUFF_recommendedDInSize();
+ ress.srcBuffer = malloc(ress.srcBufferSize);
+ ress.dstBufferSize = ZBUFF_recommendedDOutSize();
+ ress.dstBuffer = malloc(ress.dstBufferSize);
+ if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+
+ /* dictionary */
+ ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
+
+ return ress;
+}
+
+static void FIO_freeDResources(dRess_t ress)
+{
+ size_t errorCode = ZBUFF_freeDCtx(ress.dctx);
+ if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
+ free(ress.srcBuffer);
+ free(ress.dstBuffer);
+ free(ress.dictBuffer);
+}
+
+
+unsigned long long FIO_decompressFrame(dRess_t ress,
+ FILE* foutput, FILE* finput, size_t alreadyLoaded)
{
U64 frameSize = 0;
size_t readSize=alreadyLoaded;
/* Main decompression Loop */
- ZBUFF_decompressInit(dctx);
+ ZBUFF_decompressInit(ress.dctx);
+ ZBUFF_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1)
{
/* Decode */
size_t sizeCheck;
- size_t inSize=readSize, decodedSize=outBuffSize;
- size_t inStart=0;
- size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize);
+ size_t inSize=readSize, decodedSize=ress.dstBufferSize;
+ size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
readSize -= inSize;
- inStart += inSize;
/* Write block */
- sizeCheck = fwrite(outBuff, 1, decodedSize, foutput);
+ sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break;
- if (readSize) continue; /* still some data left within inBuff */
+ if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
/* Fill input buffer */
- if (toRead > inBuffSize) EXM_THROW(34, "too large block");
- readSize = fread(inBuff, 1, toRead, finput);
+ if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
+ readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize != toRead) EXM_THROW(35, "Read error");
}
@@ -359,61 +557,96 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
}
-unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+static int FIO_decompressFile_extRess(dRess_t ress,
+ const char* dstFileName, const char* srcFileName)
{
- FILE* finput, *foutput;
- BYTE* inBuff=NULL;
- size_t inBuffSize = ZBUFF_recommendedDInSize();
- BYTE* outBuff=NULL;
- size_t outBuffSize = ZBUFF_recommendedDOutSize();
- U64 filesize = 0;
- size_t toRead;
-
+ unsigned long long filesize = 0;
+ FILE* srcFile;
+ FILE* dstFile;
/* Init */
- ZBUFF_DCtx* dctx = ZBUFF_createDCtx();
- FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-
- /* Allocate Memory (if needed) */
- inBuff = (BYTE*)malloc(inBuffSize);
- outBuff = (BYTE*)malloc(outBuffSize);
- if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
+ if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
/* for each frame */
for ( ; ; )
{
- toRead = 0;
-
+ size_t sizeCheck;
+ /* check magic number -> version */
+ size_t toRead = 4;
+ sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
+ if (sizeCheck==0) break; /* no more input */
+ if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+ if (ZSTD_isLegacy(MEM_readLE32(ress.srcBuffer)))
{
- size_t sizeCheck;
- /* check magic number -> version */
- toRead = 4;
- sizeCheck = fread(inBuff, (size_t)1, toRead, finput);
- if (sizeCheck==0) break; /* no more input */
- if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
- if (ZSTD_isLegacy(MEM_readLE32(inBuff)))
- {
- filesize += FIO_decompressLegacyFrame(foutput, finput, MEM_readLE32(inBuff));
- continue;
- }
+ filesize += FIO_decompressLegacyFrame(dstFile, srcFile, MEM_readLE32(ress.srcBuffer));
+ continue;
}
#endif /* ZSTD_LEGACY_SUPPORT */
- filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, toRead, outBuff, outBuffSize, dctx);
+ filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead);
}
+ /* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize);
+ DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
- /* clean */
- free(inBuff);
- free(outBuff);
- ZBUFF_freeDCtx(dctx);
- fclose(finput);
- if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
+ /* Close */
+ fclose(srcFile);
+ if (fclose(dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
- return filesize;
+ return 0;
}
+int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
+ const char* dictFileName)
+{
+ int missingFiles = 0;
+ dRess_t ress = FIO_createDResources(dictFileName);
+
+ missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName);
+
+ FIO_freeDResources(ress);
+ return missingFiles;
+}
+
+
+#define MAXSUFFIXSIZE 8
+int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
+ const char* suffix,
+ const char* dictFileName)
+{
+ unsigned u;
+ int skippedFiles = 0;
+ int missingFiles = 0;
+ char* dstFileName = (char*)malloc(FNSPACE);
+ size_t dfnSize = FNSPACE;
+ const size_t suffixSize = strlen(suffix);
+ dRess_t ress;
+
+ if (dstFileName==NULL) EXM_THROW(70, "not enough memory for dstFileName");
+ ress = FIO_createDResources(dictFileName);
+
+ for (u=0; u<nbFiles; u++)
+ {
+ const char* srcFileName = srcNamesTable[u];
+ size_t sfnSize = strlen(srcFileName);
+ const char* suffixPtr = srcFileName + sfnSize - suffixSize;
+ if (dfnSize <= sfnSize-suffixSize+1) { free(dstFileName); dfnSize = sfnSize + 20; dstFileName = (char*)malloc(dfnSize); if (dstFileName==NULL) EXM_THROW(71, "not enough memory for dstFileName"); }
+ if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0)
+ {
+ DISPLAYLEVEL(1, "File extension doesn't match expected extension (%4s); will not process file: %s\n", suffix, srcFileName);
+ skippedFiles++;
+ continue;
+ }
+ memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
+ dstFileName[sfnSize-suffixSize] = '\0';
+
+ missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName);
+ }
+
+ FIO_freeDResources(ress);
+ free(dstFileName);
+ return missingFiles + skippedFiles;
+}
diff --git a/programs/fileio.h b/programs/fileio.h
index 037c819..0e25d84 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -50,19 +50,35 @@ void FIO_setNotificationLevel(unsigned level);
/* *************************************
-* Stream/File functions
+* Single File functions
***************************************/
-unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, int compressionLevel);
-unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename);
+int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel);
+int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
/**
FIO_compressFilename :
- @result : size of compressed file
+ @result : 0 == ok; 1 == pb with src file.
FIO_decompressFilename :
- @result : size of regenerated file
+ @result : 0 == ok; 1 == pb with src file.
*/
+/* *************************************
+* Multiple File functions
+***************************************/
+int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
+ const char* suffix,
+ const char* dictFileName, int compressionLevel);
+int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
+ const char* suffix,
+ const char* dictFileName);
+/**
+FIO_compressMultipleFilenames :
+ @result : nb of missing files
+FIO_decompressMultipleFilenames :
+ @result : nb of missing or skipped files
+*/
+
#if defined (__cplusplus)
}
diff --git a/programs/fullbench.c b/programs/fullbench.c
index cb48220..0eeb88e 100644
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -235,7 +235,7 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
- U32 DTableML[1<<11], DTableLL[1<<10], DTableOffb[1<<9];
+ U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */
const BYTE* dumps;
size_t length;
int nbSeq;
@@ -245,7 +245,6 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
-
/*********************************************************
* Bench functions
*********************************************************/
@@ -429,7 +428,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
return 11;
}
- // Memory allocation & restrictions
+ /* Memory allocation & restrictions */
inFileSize = BMK_GetFileSize(inFileName);
benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 793af7f..f3fda45 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -79,11 +79,13 @@ static const U32 prime2 = 2246822519U;
static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if ((FUZ_GetMilliSpan(g_time) > g_refreshRate) || (g_displayLevel>=4)) \
- { g_time = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
+ if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
+ { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const U32 g_refreshRate = 150;
-static U32 g_time = 0;
+static U32 g_displayTime = 0;
+
+static U32 g_testTime = 0;
/*********************************************************
@@ -259,6 +261,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
+ BYTE* mirrorBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
@@ -266,17 +269,22 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* ctx;
+ ZSTD_DCtx* dctx;
+ U32 startTime = FUZ_GetMilliStart();
/* allocation */
ctx = ZSTD_createCCtx();
+ dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
+ mirrorBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
- CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !dstBuffer || !cBuffer || !ctx,
+ CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
+ || !dstBuffer || !mirrorBuffer || !cBuffer || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
@@ -292,17 +300,23 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
FUZ_rand(&coreSeed);
/* test loop */
- for ( ; testNb <= nbTests; testNb++ )
+ for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
- size_t sampleSize, sampleStart;
- size_t cSize, dSize, dSupSize;
- U32 sampleSizeLog, buffNb, cLevelMod;
+ size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
+ size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
+ U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
+ XXH64_state_t crc64;
U64 crcOrig, crcDest;
int cLevel;
BYTE* sampleBuffer;
+ const BYTE* dict;
+ size_t dictSize;
/* init */
- DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
+ if (nbTests >= testNb)
+ { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
+ else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
+
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
buffNb = FUZ_rand(&lseed) & 127;
@@ -342,7 +356,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
/* compression failure test : too small dest buffer */
if (cSize > 3)
{
- size_t errorCode;
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
static const U32 endMark = 0x4DC2B1A9;
@@ -365,7 +378,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
/* truncated src decompression test */
{
- size_t errorCode;
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
@@ -379,7 +391,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
/* too small dst decompression test */
if (sampleSize > 3)
{
- size_t errorCode;
const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = sampleSize - missing;
static const BYTE token = 0xA9;
@@ -424,7 +435,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
U32 noiseSrc = FUZ_rand(&lseed) % 5;
const U32 endMark = 0xA9B1C3D6;
U32 endCheck;
- size_t errorCode;
srcBuffer = cNoiseBuffer[noiseSrc];
memcpy(dstBuffer+sampleSize, &endMark, 4);
errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
@@ -435,11 +445,80 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
}
}
+
+ /* Streaming compression of scattered segments test */
+ XXH64_reset(&crc64, 0);
+ nbChunks = (FUZ_rand(&lseed) & 127) + 2;
+ sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
+ maxTestSize = (size_t)1 << sampleSizeLog;
+ maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
+ if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
+
+ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
+ sampleSize = (size_t)1 << sampleSizeLog;
+ sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+ sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+ dict = srcBuffer + sampleStart;
+ dictSize = sampleSize;
+
+ cSize = ZSTD_compressBegin(ctx, cBuffer, cBufferSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
+ errorCode = ZSTD_compress_insertDictionary(ctx, dict, dictSize);
+ CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode));
+ totalTestSize = 0;
+ for (n=0; n<nbChunks; n++)
+ {
+ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
+ sampleSize = (size_t)1 << sampleSizeLog;
+ sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+ sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+
+ if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
+ /* avoid invalid dstBufferTooSmall */
+ break;
+ if (totalTestSize+sampleSize > maxTestSize) break;
+
+ errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
+ CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
+ cSize += errorCode;
+
+ XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize);
+ memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
+ totalTestSize += sampleSize;
+ }
+ errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
+ CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
+ cSize += errorCode;
+ crcOrig = XXH64_digest(&crc64);
+
+ /* streaming decompression test */
+ errorCode = ZSTD_resetDCtx(dctx);
+ CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
+ ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
+ totalCSize = 0;
+ totalGenSize = 0;
+ while (totalCSize < cSize)
+ {
+ size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
+ size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
+ CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
+ totalGenSize += genSize;
+ totalCSize += inSize;
+ }
+ CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
+ CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
+ CHECK (totalCSize != cSize, "compressed data should be fully read")
+ crcDest = XXH64(dstBuffer, totalTestSize, 0);
+ if (crcDest!=crcOrig)
+ errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
+ CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
+ (U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);
+
}
- DISPLAY("\rAll fuzzer tests completed \n");
+ DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup:
ZSTD_freeCCtx(ctx);
+ ZSTD_freeDCtx(dctx);
free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]);
@@ -447,6 +526,7 @@ _cleanup:
free(cNoiseBuffer[4]);
free(cBuffer);
free(dstBuffer);
+ free(mirrorBuffer);
return result;
_output_error:
@@ -520,7 +600,7 @@ int main(int argc, char** argv)
break;
case 'i':
- argument++;
+ argument++; g_testTime=0;
nbTests=0;
while ((*argument>='0') && (*argument<='9'))
{
@@ -530,6 +610,20 @@ int main(int argc, char** argv)
}
break;
+ case 'T':
+ argument++;
+ nbTests=0; g_testTime=0;
+ while ((*argument>='0') && (*argument<='9'))
+ {
+ g_testTime *= 10;
+ g_testTime += *argument - '0';
+ argument++;
+ }
+ if (*argument=='m') g_testTime *=60, argument++;
+ if (*argument=='n') argument++;
+ g_testTime *= 1000;
+ break;
+
case 's':
argument++;
seed=0;
@@ -580,8 +674,6 @@ int main(int argc, char** argv)
DISPLAY("Seed = %u\n", seed);
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
- if (nbTests<=0) nbTests=1;
-
if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
diff --git a/programs/paramgrill.c b/programs/paramgrill.c
index 346a9c3..a34da88 100644
--- a/programs/paramgrill.c
+++ b/programs/paramgrill.c
@@ -342,6 +342,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
+ NULL, 0,
params);
nbLoops++;
}
@@ -681,6 +682,8 @@ static void BMK_selectRandomStart(
/* totally random entry */
ZSTD_parameters p;
potentialRandomParams(&p, 1);
+ p.srcSize = srcSize;
+ ZSTD_validateParams(&p);
playAround(f, winners, p, srcBuffer, srcSize, ctx);
}
else
@@ -734,6 +737,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
for (i=1; i<=maxSeeds; i++)
{
params = ZSTD_getParams(i, blockSize);
+ ZSTD_validateParams(¶ms);
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
}
}
diff --git a/programs/playTests.sh b/programs/playTests.sh
new file mode 100755
index 0000000..5d641ec
--- /dev/null
+++ b/programs/playTests.sh
@@ -0,0 +1,104 @@
+#!/bin/sh -e
+
+die() {
+ echo "$@" 1>&2
+ exit 1
+}
+
+roundTripTest() {
+ if [ -n "$3" ]; then
+ local c="$3"
+ local p="$2"
+ else
+ local c="$2"
+ fi
+
+ rm -f tmp1 tmp2
+ echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
+ ./datagen $1 $p | md5sum > tmp1
+ ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d | md5sum > tmp2
+ diff -q tmp1 tmp2
+}
+
+[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
+
+printf "\n**** frame concatenation **** "
+
+echo "hello " > hello.tmp
+echo "world!" > world.tmp
+cat hello.tmp world.tmp > helloworld.tmp
+$ZSTD hello.tmp > hello.zstd
+$ZSTD world.tmp > world.zstd
+cat hello.zstd world.zstd > helloworld.zstd
+$ZSTD -df helloworld.zstd > result.tmp
+cat result.tmp
+sdiff helloworld.tmp result.tmp
+rm ./*.tmp ./*.zstd
+
+echo frame concatenation test completed
+
+echo "**** flush write error test **** "
+
+echo "echo foo | $ZSTD > /dev/full"
+echo foo | $ZSTD > /dev/full && die "write error not detected!"
+echo "echo foo | $ZSTD | $ZSTD -d > /dev/full"
+echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
+
+
+echo "*** dictionary tests *** "
+
+./datagen > tmpDict
+./datagen -g1M | md5sum > tmp1
+./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2
+diff -q tmp1 tmp2
+
+echo "*** multiple files tests *** "
+
+./datagen -s1 > tmp1 2> /dev/null
+./datagen -s2 -g100K > tmp2 2> /dev/null
+./datagen -s3 -g1M > tmp3 2> /dev/null
+$ZSTD -f -m tmp*
+ls -ls tmp*
+rm tmp1 tmp2 tmp3
+$ZSTD -df -m *.zst
+ls -ls tmp*
+$ZSTD -f -m tmp1 notHere tmp2 && die "missing file not detected!"
+rm tmp*
+
+echo "**** zstd round-trip tests **** "
+
+roundTripTest
+roundTripTest '' 6
+
+if [ "$1" != "--test-large-data" ]; then
+ echo "Skipping large data tests"
+ exit 0
+fi
+
+roundTripTest -g270000000 1
+roundTripTest -g270000000 2
+roundTripTest -g270000000 3
+
+roundTripTest -g140000000 -P60 4
+roundTripTest -g140000000 -P60 5
+roundTripTest -g140000000 -P60 6
+
+roundTripTest -g70000000 -P70 7
+roundTripTest -g70000000 -P70 8
+roundTripTest -g70000000 -P70 9
+
+roundTripTest -g35000000 -P75 10
+roundTripTest -g35000000 -P75 11
+roundTripTest -g35000000 -P75 12
+
+roundTripTest -g18000000 -P80 13
+roundTripTest -g18000000 -P80 14
+roundTripTest -g18000000 -P80 15
+roundTripTest -g18000000 -P80 16
+roundTripTest -g18000000 -P80 17
+
+roundTripTest -g50000000 -P94 18
+roundTripTest -g50000000 -P94 19
+
+roundTripTest -g99000000 -P99 20
+roundTripTest -g6000000000 -P99 q
diff --git a/programs/zbufftest.c b/programs/zbufftest.c
index 5903935..ab8aa34 100644
--- a/programs/zbufftest.c
+++ b/programs/zbufftest.c
@@ -161,6 +161,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZBUFF_compressInit(zc, 1);
readSize = CNBufferSize;
genSize = compressedBufferSize;
+ ZBUFF_compressWithDictionary(zc, CNBuffer, 128 KB);
result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
if (ZBUFF_isError(result)) goto _output_error;
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
@@ -174,6 +175,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_decompressInit(zd);
+ ZBUFF_decompressWithDictionary(zd, CNBuffer, 128 KB);
readSize = cSize;
genSize = CNBufferSize;
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
@@ -244,7 +246,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
U32 coreSeed = seed, lseed = 0;
ZBUFF_CCtx* zc;
ZBUFF_DCtx* zd;
- XXH64_state_t crc64;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
@@ -280,10 +281,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart;
- size_t cSize;
+ const BYTE* dict;
+ size_t cSize, dictSize;
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
size_t errorCode;
U32 sampleSizeLog, buffNb, n, nbChunks;
+ XXH64_state_t crc64;
U64 crcOrig, crcDest;
/* init */
@@ -316,6 +319,15 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
+
+ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
+ sampleSize = (size_t)1 << sampleSizeLog;
+ sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+ sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+ dict = srcBuffer + sampleStart;
+ dictSize = sampleSize;
+ ZBUFF_compressWithDictionary(zc, dict, dictSize);
+
totalTestSize = 0;
cSize = 0;
for (n=0; n<nbChunks; n++)
@@ -363,6 +375,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
/* multi - fragments decompression test */
ZBUFF_decompressInit(zd);
+ ZBUFF_decompressWithDictionary(zd, dict, dictSize);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
diff --git a/programs/zstd.1 b/programs/zstd.1
index fdc8cc4..8d69c4d 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -31,12 +31,21 @@ is equivalent to
It is based on the \fBLZ77\fR family, with FSE & huff0 entropy stage.
zstd offers compression speed > 200 MB/s per core.
It also features a fast decoder, with speed > 500 MB/s per core.
+
+\fBzstd\fR command line is generally similar to gzip, but features the following differences :
+ - Original files are preserved
+ - By default, \fBzstd file1 file2\fR means : compress file1 \fBinto\fR file2.
+ Use \fB-m\fR command if you want : compress file1 into file1.zstd and file2 into file2.zst
+ - By default, when compressing files, \fBzstd\fR displays advancement notification and result summary.
+ Use \fB-q\fR to turn them off
+
+
\fBzstd\fR supports the following options :
.SH OPTIONS
.TP
-.B \-1
- fast compression (default)
+.B \-#
+ # compression level [1-19](default:1)
.TP
.B \-d
decompression
@@ -44,6 +53,14 @@ It also features a fast decoder, with speed > 500 MB/s per core.
.B \-f
overwrite output without prompting
.TP
+.BR \-m ", " --multiple
+ multiple files mode
+ In this mode, multiple files on the command line means compression or decompression of each named file
+ Notifications are also turned off by default
+.TP
+.B \-D
+ Use next file as dictionary content for compress / decompression
+.TP
.BR \-h/\-H ", " --help
display help/long help and exit
.TP
@@ -53,20 +70,17 @@ It also features a fast decoder, with speed > 500 MB/s per core.
.BR \-v ", " --verbose
verbose mode
.TP
-.B \-q
- suppress warnings; specify twice to suppress errors too
+.BR \-q ", " --quiet
+ suppress warnings and notifications; specify twice to suppress errors too
.TP
-.B \-c
+.B \-c
force write to standard output, even if it is the console
.TP
-.B \-t
- test compressed file integrity
-.TP
.B \-z
force compression
.TP
-.B \-b
- benchmark file(s)
+.B \-b#
+ benchmark file(s) using compression level #
.TP
.B \-i#
iteration loops [1-9](default : 3), benchmark mode only
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 418fec7..564686c 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -43,23 +43,26 @@
#include <stdio.h> /* fprintf, getchar */
#include <stdlib.h> /* exit, calloc, free */
#include <string.h> /* strcmp, strlen */
-#include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */
#include "fileio.h"
+#ifndef ZSTD_NOBENCH
+# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */
+#endif
+#include "zstd.h" /* ZSTD version numbers */
/**************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-# include <fcntl.h> // _O_BINARY
-# include <io.h> // _setmode, _isatty
+# include <fcntl.h> /* _O_BINARY */
+# include <io.h> /* _setmode, _isatty */
# ifdef __MINGW32__
/* int _fileno(FILE *stream); // seems no longer useful // MINGW somehow forgets to include this windows declaration into <stdio.h> */
# endif
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
-# include <unistd.h> // isatty
+# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#endif
@@ -70,7 +73,9 @@
**************************************/
#define COMPRESSOR_NAME "zstd command line interface"
#ifndef ZSTD_VERSION
-# define ZSTD_VERSION "v0.4.0"
+# define QUOTE(str) #str
+# define EXPAND_AND_QUOTE(str) QUOTE(str)
+# define ZSTD_VERSION "v" EXPAND_AND_QUOTE(ZSTD_VERSION_MAJOR) "." EXPAND_AND_QUOTE(ZSTD_VERSION_MINOR) "." EXPAND_AND_QUOTE(ZSTD_VERSION_RELEASE)
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__
@@ -118,8 +123,7 @@ static int usage(const char* programName)
DISPLAY( "input : a filename\n");
DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n");
- DISPLAY( " -1 : Fast compression (default) \n");
- DISPLAY( " -9 : High compression \n");
+ DISPLAY( " -# : # compression level (1-19, default:1) \n");
DISPLAY( " -d : decompression (default for %s extension)\n", ZSTD_EXTENSION);
//DISPLAY( " -z : force compression\n");
DISPLAY( " -f : overwrite output without prompting \n");
@@ -136,13 +140,16 @@ static int usage_advanced(const char* programName)
DISPLAY( " -V : display Version number and exit\n");
DISPLAY( " -v : verbose mode\n");
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
+ DISPLAY( " -m : multiple input filenames mode \n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
- //DISPLAY( " -t : test compressed file integrity\n");
+ DISPLAY( " -D file: use file content as Dictionary \n");
+#ifndef ZSTD_NOBENCH
DISPLAY( "Benchmark arguments :\n");
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
DISPLAY( " -B# : cut file into independent blocks of size # (default : no block)\n");
DISPLAY( " -i# : iteration loops [1-9](default : 3)\n");
DISPLAY( " -r# : test all compression levels from 1 to # (default : disabled)\n");
+#endif
return 0;
}
@@ -170,31 +177,31 @@ int main(int argCount, const char** argv)
decode=0,
forceStdout=0,
main_pause=0,
- rangeBench = 1;
- unsigned fileNameStart = 0;
- unsigned nbFiles = 0;
+ nextEntryIsDictionary=0,
+ multiple=0,
+ operationResult=0;
unsigned cLevel = 1;
+ const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
+ unsigned filenameIdx = 0;
const char* programName = argv[0];
- const char* inFileName = NULL;
const char* outFileName = NULL;
+ const char* dictFileName = NULL;
char* dynNameSpace = NULL;
const char extension[] = ZSTD_EXTENSION;
+ int rangeBench = 1;
+ /* init */
+ (void)rangeBench; /* not used when ZSTD_NOBENCH set */
+ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
displayOut = stderr;
- /* Pick out basename component. Don't rely on stdlib because of conflicting behavior. */
- for (i = (int)strlen(programName); i > 0; i--)
- {
- if (programName[i] == '/') { i++; break; }
- }
+ /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
+ for (i = (int)strlen(programName); i > 0; i--) { if (programName[i] == '/') { i++; break; } }
programName += i;
- /* zstdcat preset behavior */
+ /* preset behaviors */
+ if (!strcmp(programName, ZSTD_UNZSTD)) decode=1;
if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
- /* unzstd preset behavior */
- if (!strcmp(programName, ZSTD_UNZSTD))
- decode=1;
-
/* command switches */
for(i=1; i<argCount; i++)
{
@@ -205,7 +212,9 @@ int main(int argCount, const char** argv)
/* long commands (--long-word) */
if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--help")) { displayOut=stdout; return usage_advanced(programName); }
+ if (!strcmp(argument, "--multiple")) { multiple=1; continue; }
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
+ if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
/* Decode commands (note : aggregated commands are allowed) */
if (argument[0]=='-')
@@ -213,9 +222,8 @@ int main(int argCount, const char** argv)
/* '-' means stdin/stdout */
if (argument[1]==0)
{
- if (!inFileName) inFileName=stdinmark;
- else outFileName=stdoutmark;
- continue;
+ if (!filenameIdx) { filenameIdx=1, filenameTable[0]=stdinmark; continue; }
+ outFileName=stdoutmark; continue;
}
argument++;
@@ -248,11 +256,17 @@ int main(int argCount, const char** argv)
/* Decoding */
case 'd': decode=1; argument++; break;
+ /* Multiple input files */
+ case 'm': multiple=1; argument++; break;
+
/* Force stdout, even if stdout==console */
case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel=1; argument++; break;
- // Test
- //case 't': decode=1; LZ4IO_setOverwrite(1); output_filename=nulmark; break;
+ /* Use file content as dictionary */
+ case 'D': nextEntryIsDictionary = 1; argument++; break;
+
+ /* Test -- not implemented */
+ /* case 't': decode=1; LZ4IO_setOverwrite(1); output_filename=nulmark; break; */
/* Overwrite */
case 'f': FIO_overwriteMode(); argument++; break;
@@ -263,9 +277,10 @@ int main(int argCount, const char** argv)
/* Quiet mode */
case 'q': displayLevel--; argument++; break;
- /* keep source file (default anyway, so useless; only for xz/lzma compatibility) */
+ /* keep source file (default anyway, so useless; for gzip/xz compatibility) */
case 'k': argument++; break;
+#ifndef ZSTD_NOBENCH
/* Benchmark */
case 'b': bench=1; argument++; break;
@@ -299,6 +314,7 @@ int main(int argCount, const char** argv)
rangeBench = -1;
argument++;
break;
+#endif /* ZSTD_NOBENCH */
/* Pause at the end (hidden option) */
case 'p': main_pause=1; argument++; break;
@@ -310,40 +326,47 @@ int main(int argCount, const char** argv)
continue;
}
- /* first provided filename is input */
- if (!inFileName) { inFileName = argument; fileNameStart = i; nbFiles = argCount-i; continue; }
-
- /* second provided filename is output */
- if (!outFileName)
+ /* dictionary */
+ if (nextEntryIsDictionary)
{
- outFileName = argument;
- if (!strcmp (outFileName, nullString)) outFileName = nulmark;
+ nextEntryIsDictionary = 0;
+ dictFileName = argument;
continue;
}
+
+ /* add filename to list */
+ filenameTable[filenameIdx++] = argument;
}
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
+ /* Check if benchmark is selected */
+ if (bench)
+ {
+#ifndef ZSTD_NOBENCH
+ BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel*rangeBench);
+#endif
+ goto _end;
+ }
+
/* No input filename ==> use stdin */
- if(!inFileName) { inFileName=stdinmark; }
+ if(!filenameIdx) filenameIdx=1, filenameTable[0]=stdinmark;
/* Check if input defined as console; trigger an error in this case */
- if (!strcmp(inFileName, stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
-
- /* Check if benchmark is selected */
- if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, cLevel*rangeBench); goto _end; }
+ if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
/* No output filename ==> try to select one automatically (when possible) */
- while (!outFileName)
+ if (filenameIdx>=2) outFileName = filenameTable[1];
+ while (!outFileName) /* while : just to allow break statement */
{
if (!IS_CONSOLE(stdout)) { outFileName=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */
if (!decode) /* compression to file */
{
- size_t l = strlen(inFileName);
+ size_t l = strlen(filenameTable[0]);
dynNameSpace = (char*)calloc(1,l+5);
if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); }
- strcpy(dynNameSpace, inFileName);
+ strcpy(dynNameSpace, filenameTable[0]);
strcpy(dynNameSpace+l, ZSTD_EXTENSION);
outFileName = dynNameSpace;
DISPLAYLEVEL(2, "Compressed filename will be : %s \n", outFileName);
@@ -351,8 +374,8 @@ int main(int argCount, const char** argv)
}
/* decompression to file (automatic name will work only if input filename has correct format extension) */
{
- size_t filenameSize = strlen(inFileName);
- if (strcmp(inFileName + (filenameSize-4), extension))
+ size_t filenameSize = strlen(filenameTable[0]);
+ if (strcmp(filenameTable[0] + (filenameSize-4), extension))
{
DISPLAYLEVEL(1, "unknown suffix - cannot determine destination filename\n");
return badusage(programName);
@@ -360,7 +383,7 @@ int main(int argCount, const char** argv)
dynNameSpace = (char*)calloc(1,filenameSize+1);
if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); }
outFileName = dynNameSpace;
- strcpy(dynNameSpace, inFileName);
+ strcpy(dynNameSpace, filenameTable[0]);
dynNameSpace[filenameSize-4]=0;
DISPLAYLEVEL(2, "Decoding file %s \n", outFileName);
}
@@ -369,18 +392,36 @@ int main(int argCount, const char** argv)
/* Check if output is defined as console; trigger an error in this case */
if (!strcmp(outFileName,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName);
- /* No warning message in pure pipe mode (stdin + stdout) */
- if (!strcmp(inFileName, stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
+ /* No warning message in pure pipe mode (stdin + stdout) or multiple mode */
+ if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
+ if (multiple && (displayLevel==2)) displayLevel=1;
+
+ if ((!multiple) && (filenameIdx>2))
+ {
+ DISPLAY("Too many files on the command line (%u > 2). Do you mean -m ? \n", filenameIdx);
+ return filenameIdx;
+ }
/* IO Stream/File */
FIO_setNotificationLevel(displayLevel);
if (decode)
- FIO_decompressFilename(outFileName, inFileName);
+ {
+ if (multiple)
+ operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName);
+ else
+ operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
+ }
else
- FIO_compressFilename(outFileName, inFileName, cLevel);
+ {
+ if (multiple)
+ operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName, cLevel);
+ else
+ operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
+ }
_end:
if (main_pause) waitEnter();
free(dynNameSpace);
- return 0;
+ free((void*)filenameTable);
+ return operationResult;
}
diff --git a/visual/2012/fullbench/fullbench.vcxproj b/visual/2013/fullbench/fullbench.vcxproj
similarity index 94%
rename from visual/2012/fullbench/fullbench.vcxproj
rename to visual/2013/fullbench/fullbench.vcxproj
index 9a36e5b..c0d7376 100644
--- a/visual/2012/fullbench/fullbench.vcxproj
+++ b/visual/2013/fullbench/fullbench.vcxproj
@@ -27,26 +27,26 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
@@ -111,7 +111,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
- <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+ <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -129,7 +129,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>true</EnablePREfast>
<TreatWarningAsError>true</TreatWarningAsError>
- <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+ <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -149,7 +149,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
- <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+ <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
diff --git a/visual/2012/fullbench/fullbench.vcxproj.filters b/visual/2013/fullbench/fullbench.vcxproj.filters
similarity index 100%
rename from visual/2012/fullbench/fullbench.vcxproj.filters
rename to visual/2013/fullbench/fullbench.vcxproj.filters
diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2013/fuzzer/fuzzer.vcxproj
similarity index 94%
rename from visual/2012/fuzzer/fuzzer.vcxproj
rename to visual/2013/fuzzer/fuzzer.vcxproj
index 63ab472..0844efe 100644
--- a/visual/2012/fuzzer/fuzzer.vcxproj
+++ b/visual/2013/fuzzer/fuzzer.vcxproj
@@ -27,26 +27,26 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
@@ -111,7 +111,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
- <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+ <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -129,7 +129,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>true</EnablePREfast>
<TreatWarningAsError>true</TreatWarningAsError>
- <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+ <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -149,7 +149,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
- <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+ <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
diff --git a/visual/2012/fuzzer/fuzzer.vcxproj.filters b/visual/2013/fuzzer/fuzzer.vcxproj.filters
similarity index 100%
rename from visual/2012/fuzzer/fuzzer.vcxproj.filters
rename to visual/2013/fuzzer/fuzzer.vcxproj.filters
diff --git a/visual/2012/zstd.sln b/visual/2013/zstd.sln
similarity index 74%
rename from visual/2012/zstd.sln
rename to visual/2013/zstd.sln
index 2b5eb98..4f2447f 100644
--- a/visual/2012/zstd.sln
+++ b/visual/2013/zstd.sln
@@ -1,12 +1,16 @@
Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Express 2012 for Windows Desktop
+# Visual Studio 14
+VisualStudioVersion = 14.0.24720.0
+MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcxproj", "{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{6FD4352B-346C-4703-96EA-D4A8B9A6976E}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}"
EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
@@ -39,6 +43,14 @@ Global
{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.Build.0 = Release|Win32
{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.ActiveCfg = Release|x64
{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.Build.0 = Release|x64
+ {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.ActiveCfg = Debug|Win32
+ {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.Build.0 = Debug|Win32
+ {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.ActiveCfg = Debug|x64
+ {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.Build.0 = Debug|x64
+ {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.ActiveCfg = Release|Win32
+ {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32
+ {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64
+ {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
diff --git a/visual/2012/zstd/zstd.vcxproj b/visual/2013/zstd/zstd.vcxproj
similarity index 94%
copy from visual/2012/zstd/zstd.vcxproj
copy to visual/2013/zstd/zstd.vcxproj
index f952462..2ab1285 100644
--- a/visual/2012/zstd/zstd.vcxproj
+++ b/visual/2013/zstd/zstd.vcxproj
@@ -28,6 +28,7 @@
<ClCompile Include="..\..\..\lib\zstd_compress.c" />
<ClCompile Include="..\..\..\lib\zstd_decompress.c" />
<ClCompile Include="..\..\..\programs\bench.c" />
+ <ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\fileio.c" />
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
<ClCompile Include="..\..\..\programs\xxhash.c" />
@@ -48,6 +49,7 @@
<ClInclude Include="..\..\..\lib\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\zstd_static.h" />
<ClInclude Include="..\..\..\programs\bench.h" />
+ <ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\fileio.h" />
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" />
<ClInclude Include="..\..\..\programs\xxhash.h" />
@@ -61,27 +63,27 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
@@ -102,7 +104,7 @@
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
- <IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
diff --git a/visual/2012/zstd/zstd.vcxproj.filters b/visual/2013/zstd/zstd.vcxproj.filters
similarity index 92%
rename from visual/2012/zstd/zstd.vcxproj.filters
rename to visual/2013/zstd/zstd.vcxproj.filters
index 442ecae..49bae35 100644
--- a/visual/2012/zstd/zstd.vcxproj.filters
+++ b/visual/2013/zstd/zstd.vcxproj.filters
@@ -54,6 +54,9 @@
<ClCompile Include="..\..\..\lib\legacy\zstd_v03.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
+ <ClCompile Include="..\..\..\programs\datagen.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\fse.h">
@@ -107,5 +110,8 @@
<ClInclude Include="..\..\..\lib\legacy\zstd_v03.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
+ <ClInclude Include="..\..\..\programs\datagen.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
</ItemGroup>
</Project>
\ No newline at end of file
diff --git a/visual/2013/zstdlib/resource.h b/visual/2013/zstdlib/resource.h
new file mode 100644
index 0000000..db984ef
Binary files /dev/null and b/visual/2013/zstdlib/resource.h differ
diff --git a/visual/2013/zstdlib/zstdlib.rc b/visual/2013/zstdlib/zstdlib.rc
new file mode 100644
index 0000000..d5b107e
Binary files /dev/null and b/visual/2013/zstdlib/zstdlib.rc differ
diff --git a/visual/2012/zstd/zstd.vcxproj b/visual/2013/zstdlib/zstdlib.vcxproj
similarity index 68%
rename from visual/2012/zstd/zstd.vcxproj
rename to visual/2013/zstdlib/zstdlib.vcxproj
index f952462..1245955 100644
--- a/visual/2012/zstd/zstd.vcxproj
+++ b/visual/2013/zstdlib/zstdlib.vcxproj
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
@@ -21,67 +21,58 @@
<ItemGroup>
<ClCompile Include="..\..\..\lib\fse.c" />
<ClCompile Include="..\..\..\lib\huff0.c" />
- <ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
- <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
- <ClCompile Include="..\..\..\lib\legacy\zstd_v03.c" />
<ClCompile Include="..\..\..\lib\zstd_buffered.c" />
<ClCompile Include="..\..\..\lib\zstd_compress.c" />
<ClCompile Include="..\..\..\lib\zstd_decompress.c" />
- <ClCompile Include="..\..\..\programs\bench.c" />
- <ClCompile Include="..\..\..\programs\fileio.c" />
- <ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
- <ClCompile Include="..\..\..\programs\xxhash.c" />
- <ClCompile Include="..\..\..\programs\zstdcli.c" />
</ItemGroup>
<ItemGroup>
+ <ClInclude Include="..\..\..\lib\bitstream.h" />
+ <ClInclude Include="..\..\..\lib\error.h" />
<ClInclude Include="..\..\..\lib\fse.h" />
<ClInclude Include="..\..\..\lib\fse_static.h" />
<ClInclude Include="..\..\..\lib\huff0.h" />
<ClInclude Include="..\..\..\lib\huff0_static.h" />
- <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
- <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
- <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
- <ClInclude Include="..\..\..\lib\legacy\zstd_v03.h" />
+ <ClInclude Include="..\..\..\lib\mem.h" />
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\zstd_buffered.h" />
<ClInclude Include="..\..\..\lib\zstd_buffered_static.h" />
<ClInclude Include="..\..\..\lib\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\zstd_static.h" />
- <ClInclude Include="..\..\..\programs\bench.h" />
- <ClInclude Include="..\..\..\programs\fileio.h" />
- <ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" />
- <ClInclude Include="..\..\..\programs\xxhash.h" />
+ <ClInclude Include="resource.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <ResourceCompile Include="zstdlib.rc" />
</ItemGroup>
<PropertyGroup Label="Globals">
- <ProjectGuid>{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}</ProjectGuid>
+ <ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
- <RootNamespace>zstd</RootNamespace>
+ <RootNamespace>zstdlib</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
+ <PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v110</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
+ <PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
@@ -102,21 +93,33 @@
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
- <IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ <TargetName>zstdlib_x86</TargetName>
+ <IntDir>$(Platform)\$(Configuration)\</IntDir>
+ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
+ <IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
+ <TargetName>zstdlib_x64</TargetName>
+ <IntDir>$(Platform)\$(Configuration)\</IntDir>
+ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
+ <TargetName>zstdlib_x86</TargetName>
+ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
+ <IntDir>$(Platform)\$(Configuration)\</IntDir>
<IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
- <RunCodeAnalysis>true</RunCodeAnalysis>
+ <RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
+ <TargetName>zstdlib_x64</TargetName>
+ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
+ <IntDir>$(Platform)\$(Configuration)\</IntDir>
<IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
@@ -126,7 +129,11 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <MinimalRebuild>true</MinimalRebuild>
+ <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
+ <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+ <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
@@ -134,6 +141,7 @@
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
+ <TargetMachine>MachineX86</TargetMachine>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@@ -142,10 +150,13 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
+ <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
+ <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<EnablePREfast>true</EnablePREfast>
- <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+ <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -160,16 +171,19 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>true</EnablePREfast>
- <TreatWarningAsError>true</TreatWarningAsError>
- <AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
+ <AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
+ <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+ <AssemblerOutput>All</AssemblerOutput>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
+ <TargetMachine>MachineX86</TargetMachine>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
@@ -180,10 +194,14 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <OmitFramePointers>true</OmitFramePointers>
+ <AssemblerOutput>All</AssemblerOutput>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
diff --git a/visual/2013/zstdlib/zstdlib.vcxproj.filters b/visual/2013/zstdlib/zstdlib.vcxproj.filters
new file mode 100644
index 0000000..9e930d7
--- /dev/null
+++ b/visual/2013/zstdlib/zstdlib.vcxproj.filters
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Source Files">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Header Files">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Resource Files">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\fse.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\huff0.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\zstd_buffered.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\zstd_compress.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\zstd_decompress.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\fse.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\fse_static.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\zstd.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\zstd_static.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\huff0.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\huff0_static.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="resource.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\error.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\bitstream.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\zstd_internal.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\zstd_buffered.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\zstd_buffered_static.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\mem.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
+ </ItemGroup>
+ <ItemGroup>
+ <ResourceCompile Include="zstdlib.rc">
+ <Filter>Resource Files</Filter>
+ </ResourceCompile>
+ </ItemGroup>
+</Project>
\ No newline at end of file
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/libzstd.git
More information about the debian-med-commit
mailing list