[Git][java-team/snappy-java][upstream] 4 commits: New upstream version 1.1.7.8
Emmanuel Bourg
gitlab at salsa.debian.org
Mon Jan 25 10:33:55 GMT 2021
Emmanuel Bourg pushed to branch upstream at Debian Java Maintainers / snappy-java
Commits:
d617012f by Emmanuel Bourg at 2021-01-25T09:52:31+01:00
New upstream version 1.1.7.8
- - - - -
ee296c41 by Emmanuel Bourg at 2021-01-25T09:52:44+01:00
New upstream version 1.1.8
- - - - -
7b635bea by Emmanuel Bourg at 2021-01-25T09:52:53+01:00
New upstream version 1.1.8.1
- - - - -
0abd6862 by Emmanuel Bourg at 2021-01-25T09:53:00+01:00
New upstream version 1.1.8.3
- - - - -
26 changed files:
- .github/workflows/snapshot.yml
- .github/workflows/test.yml
- Makefile
- Makefile.common
- Makefile.package
- Milestone.md
- README.md
- build.sbt
- docker/Makefile
- + docker/dockcross-arm64
- src/main/java/org/xerial/snappy/PureJavaCrc32C.java
- src/main/java/org/xerial/snappy/SnappyFramed.java
- src/main/java/org/xerial/snappy/SnappyFramedInputStream.java
- src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java
- src/main/java/org/xerial/snappy/SnappyLoader.java
- src/main/java/org/xerial/snappy/pure/PureJavaSnappy.java
- src/main/java/org/xerial/snappy/pure/SnappyRawCompressor.java
- src/main/java/org/xerial/snappy/pure/SnappyRawDecompressor.java
- src/main/java/org/xerial/snappy/pure/UnsafeUtil.java
- src/main/resources/org/xerial/snappy/VERSION
- src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java
- − src/test/resources/lib/Linux/libhadoop.so
- − src/test/resources/lib/Linux/libsnappy.so
- − src/test/resources/lib/MacOSX/libhadoop.dylib
- − src/test/resources/lib/MacOSX/libsnappy.dylib
- version.sbt
Changes:
=====================================
.github/workflows/snapshot.yml
=====================================
@@ -22,7 +22,7 @@ jobs:
fetch-depth: 10000
# Fetch all tags so that sbt-dynver can find the previous release version
- run: git fetch --tags
- - uses: olafurpg/setup-scala at v5
+ - uses: olafurpg/setup-scala at v10
with:
java-version: adopt at 1.11
- uses: actions/cache at v1
=====================================
.github/workflows/test.yml
=====================================
@@ -29,7 +29,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout at v2
- - uses: olafurpg/setup-scala at v7
+ - uses: olafurpg/setup-scala at v10
with:
java-version: adopt at 1.11
- uses: actions/cache at v1
@@ -44,7 +44,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout at v2
- - uses: olafurpg/setup-scala at v7
+ - uses: olafurpg/setup-scala at v10
with:
java-version: adopt at 1.8
- uses: actions/cache at v1
=====================================
Makefile
=====================================
@@ -1,6 +1,8 @@
include Makefile.common
+$(info OS_NAME:$(OS_NAME), OS_ARCH:$(OS_ARCH))
+
MVN:=mvn
SBT:=./sbt
@@ -8,11 +10,11 @@ all: snappy
SNAPPY_OUT:=$(TARGET)/snappy-$(SNAPPY_VERSION)-$(os_arch)
SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(SNAPPY_VERSION).tar.gz
-SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy.cc
+SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy-c.cc snappy.cc
SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(SNAPPY_VERSION)
SNAPPY_SRC:=$(addprefix $(SNAPPY_SRC_DIR)/,$(SNAPPY_CC))
SNAPPY_GIT_REPO_URL:=https://github.com/google/snappy
-SNAPPY_GIT_REV:=b02bfa754ebf27921d8da3bd2517eab445b84ff9 # 1.1.7
+SNAPPY_GIT_REV:=537f4ad6240e586970fe554614542e9717df7902 # 1.1.8
SNAPPY_UNPACKED:=$(TARGET)/snappy-extracted.log
SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
SNAPPY_CMAKE_CACHE=$(SNAPPY_OUT)/CMakeCache.txt
@@ -144,7 +146,7 @@ native: jni-header snappy-header $(NATIVE_DLL)
native-nocmake: jni-header $(NATIVE_DLL)
snappy: native $(TARGET)/$(snappy-jar-version).jar
-native-all: win32 win64 mac64 native-arm linux32 linux64 linux-ppc64le linux-aarch64
+native-all: native mac64 win32 win64 native-arm linux32 linux64 linux-ppc64le
$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
@mkdir -p $(@D)
@@ -173,7 +175,7 @@ mac32: jni-header
$(MAKE) native OS_NAME=Mac OS_ARCH=x86
mac64: jni-header
- docker run -it $(DOCKER_RUN_OPTS) -v $$PWD:/workdir -e CROSS_TRIPLE=x86_64-apple-darwin multiarch/crossbuild make clean-native native OS_NAME=Mac OS_ARCH=x86_64
+ docker run -it $(DOCKER_RUN_OPTS) -v $$PWD:/workdir -e CROSS_TRIPLE=x86_64-apple-darwin xerial/crossbuild make clean-native native OS_NAME=Mac OS_ARCH=x86_64
linux32: jni-header
docker run $(DOCKER_RUN_OPTS) -ti -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86'
@@ -185,16 +187,16 @@ freebsd64:
$(MAKE) native OS_NAME=FreeBSD OS_ARCH=x86_64
# For ARM
-native-arm: linux-arm linux-armv6 linux-armv7 linux-android-arm
+native-arm: linux-arm linux-armv6 linux-armv7 linux-android-arm linux-arm64
linux-arm: jni-header
- ./docker/dockcross-armv5 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=arm-linux-gnueabi- OS_NAME=Linux OS_ARCH=arm'
+ ./docker/dockcross-armv5 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/armv5-unknown-linux-gnueabi/bin//armv5-unknown-linux-gnueabi- OS_NAME=Linux OS_ARCH=arm'
linux-armv6: jni-header
./docker/dockcross-armv6 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=arm-linux-gnueabihf- OS_NAME=Linux OS_ARCH=armv6'
linux-armv7: jni-header
- ./docker/dockcross-armv7 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=arm-linux-gnueabihf- OS_NAME=Linux OS_ARCH=armv7'
+ ./docker/dockcross-armv7 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/armv7-unknown-linux-gnueabi/bin/armv7-unknown-linux-gnueabi- OS_NAME=Linux OS_ARCH=armv7'
linux-android-arm: jni-header
./docker/dockcross-android-arm -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/arm-linux-androideabi/bin/arm-linux-androideabi- OS_NAME=Linux OS_ARCH=android-arm'
@@ -205,8 +207,8 @@ linux-ppc64le: jni-header
linux-ppc64: jni-header
./docker/dockcross-ppc64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=powerpc64-linux-gnu- OS_NAME=Linux OS_ARCH=ppc64'
-linux-aarch64: jni-header
- ./docker/dockcross-aarch64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=aarch64-linux-gnu- OS_NAME=Linux OS_ARCH=aarch64'
+linux-arm64: jni-header
+ ./docker/dockcross-arm64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/aarch64-unknown-linux-gnueabi/bin/aarch64-unknown-linux-gnueabi- OS_NAME=Linux OS_ARCH=aarch64'
javadoc:
$(SBT) doc
=====================================
Makefile.common
=====================================
@@ -45,7 +45,7 @@ endif
# os=Default is meant to be generic unix/linux
-known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armv6 Linux-armv7 Linux-android-arm Linux-aarch64 Linux-ppc Linux-ppc64 Linux-ppc64le Linux-s390 Linux-s390x Mac-x86 Mac-x86_64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc AIX-ppc64
+known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armv6 Linux-armv7 Linux-android-arm Linux-aarch64 Linux-ppc Linux-ppc64 Linux-ppc64le Linux-s390 Linux-s390x Mac-x86 Mac-x86_64 Mac-aarch64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc AIX-ppc64
os_arch := $(OS_NAME)-$(OS_ARCH)
IBM_JDK_7 := $(findstring IBM, $(shell $(JAVA) -version 2>&1 | grep IBM | grep "JRE 1.7"))
@@ -70,7 +70,7 @@ CROSS_PREFIX :=
Default_CXX := $(CROSS_PREFIX)g++
Default_STRIP := $(CROSS_PREFIX)strip
-Default_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
+Default_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -std=c++11
Default_LINKFLAGS := -shared -static
Default_LIBNAME := libsnappyjava.so
Default_SNAPPY_FLAGS :=
@@ -78,9 +78,9 @@ Default_SNAPPY_FLAGS :=
Linux-x86_CXX := $(CROSS_PREFIX)g++
Linux-x86_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
- Linux-x86_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__
+ Linux-x86_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__ -std=c++11
else
- Linux-x86_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__
+ Linux-x86_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m32 -U__SSE2__ -std=c++11
endif
Linux-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-x86_LIBNAME := libsnappyjava.so
@@ -89,9 +89,9 @@ Linux-x86_SNAPPY_FLAGS:=
Linux-x86_64_CXX := $(CROSS_PREFIX)g++
Linux-x86_64_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
- Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64
+ Linux-x86_64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
else
- Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64
+ Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
endif
Linux-x86_64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-x86_64_LIBNAME := libsnappyjava.so
@@ -100,9 +100,9 @@ Linux-x86_64_SNAPPY_FLAGS :=
Linux-ppc_CXX := g++
Linux-ppc_STRIP := strip
ifeq ($(IBM_JDK_7),)
- Linux-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m32
+ Linux-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m32 -std=c++11
else
- Linux-ppc_CXXFLAGS := -include lib/inc_linux/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m32
+ Linux-ppc_CXXFLAGS := -include lib/inc_linux/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m32 -std=c++11
endif
Linux-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc_LIBNAME := libsnappyjava.so
@@ -111,9 +111,9 @@ Linux-ppc_SNAPPY_FLAGS :=
Linux-ppc64le_CXX := $(CROSS_PREFIX)g++
Linux-ppc64le_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
- Linux-ppc64le_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
+ Linux-ppc64le_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
else
- Linux-ppc64le_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
+ Linux-ppc64le_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 -std=c++11
endif
# ppcle64 GLIBC is at 2.17; so disable __tls_get_addr_opt which is dependent on 2.22;
Linux-ppc64le_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -Wl,--no-tls-optimize,--no-tls-get-addr-optimize
@@ -123,9 +123,9 @@ Linux-ppc64le_SNAPPY_FLAGS :=
Linux-ppc64_CXX := $(CROSS_PREFIX)g++
Linux-ppc64_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
- Linux-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
+ Linux-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
else
- Linux-ppc64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
+ Linux-ppc64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64 -std=c++11
endif
Linux-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc64_LIBNAME := libsnappyjava.so
@@ -135,9 +135,9 @@ AIX-ppc_CXX := g++
AIX-ppc_STRIP := strip
AIX-ppc_LIBNAME := libsnappyjava.a
ifeq ($(IBM_JDK_7),)
- AIX-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix32
+ AIX-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix32 -std=c++11
else
- AIX-ppc_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix32
+ AIX-ppc_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix32 -std=c++11
endif
AIX-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc_SNAPPY_FLAGS :=
@@ -146,9 +146,9 @@ AIX-ppc64_CXX := g++
AIX-ppc64_STRIP := strip -X64
AIX-ppc64_LIBNAME := libsnappyjava.a
ifeq ($(IBM_JDK_7),)
- AIX-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix64
+ AIX-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix64 -std=c++11
else
- AIX-ppc64_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64
+ AIX-ppc64_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64 -std=c++11
endif
AIX-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc64_SNAPPY_FLAGS :=
@@ -156,9 +156,9 @@ AIX-ppc64_SNAPPY_FLAGS :=
Linux-s390_CXX := g++
Linux-s390_STRIP := strip
ifeq ($(IBM_JDK_7),)
- Linux-s390_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m31
+ Linux-s390_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m31 -std=c++11
else
- Linux-s390_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m31
+ Linux-s390_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m31 -std=c++11
endif
Linux-s390_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390_LIBNAME := libsnappyjava.so
@@ -167,9 +167,9 @@ Linux-s390_SNAPPY_FLAGS :=
Linux-s390x_CXX := g++
Linux-s390x_STRIP := strip
ifeq ($(IBM_JDK_7),)
- Linux-s390x_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
+ Linux-s390x_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 -std=c++11
else
- Linux-s390x_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m64
+ Linux-s390x_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m64 -std=c++11
endif
Linux-s390x_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390x_LIBNAME := libsnappyjava.so
@@ -203,70 +203,76 @@ SunOS-x86_64_SNAPPY_FLAGS :=
Linux-arm_CXX := $(CROSS_PREFIX)g++
Linux-arm_STRIP := $(CROSS_PREFIX)strip
-Linux-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp
+Linux-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp -std=c++11
Linux-arm_LINKFLAGS := -shared -static-libgcc
Linux-arm_LIBNAME := libsnappyjava.so
Linux-arm_SNAPPY_FLAGS:=
Linux-armv6_CXX := $(CROSS_PREFIX)g++
Linux-armv6_STRIP := $(CROSS_PREFIX)strip
-Linux-armv6_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard
-Linux-armv6_LINKFLAGS := -shared -static-libgcc
+Linux-armv6_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard -std=c++11
+Linux-armv6_LINKFLAGS := -shared -static-libgcc -std=c++11
Linux-armv6_LIBNAME := libsnappyjava.so
Linux-armv6_SNAPPY_FLAGS:=
Linux-armv7_CXX := $(CROSS_PREFIX)g++
Linux-armv7_STRIP := $(CROSS_PREFIX)strip
-Linux-armv7_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard
-Linux-armv7_LINKFLAGS := -shared -static-libgcc
+Linux-armv7_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard -std=c++11
+Linux-armv7_LINKFLAGS := -shared -static-libgcc
Linux-armv7_LIBNAME := libsnappyjava.so
Linux-armv7_SNAPPY_FLAGS:=
Linux-android-arm_CXX := $(CROSS_PREFIX)g++
Linux-android-arm_STRIP := $(CROSS_PREFIX)strip
-Linux-android-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
+Linux-android-arm_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -isystem /usr/arm-linux-androideabi/include/c++/4.9.x -std=c++11
Linux-android-arm_LINKFLAGS := -shared -static-libgcc
Linux-android-arm_LIBNAME := libsnappyjava.so
Linux-android-arm_SNAPPY_FLAGS:=
Linux-aarch64_CXX := $(CROSS_PREFIX)g++
Linux-aarch64_STRIP := $(CROSS_PREFIX)strip
-Linux-aarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
-Linux-aarch64_LINKFLAGS := -shared -static-libgcc
+Linux-aarch64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -std=c++11
+Linux-aarch64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-aarch64_LIBNAME := libsnappyjava.so
Linux-aarch64_SNAPPY_FLAGS:=
Mac-x86_CXX := g++ -arch i386
Mac-x86_STRIP := strip -x
-Mac-x86_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.4 -fvisibility=hidden
+Mac-x86_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.4 -fvisibility=hidden -std=c++11
Mac-x86_LINKFLAGS := -dynamiclib -static-libgcc
-Mac-x86_LIBNAME := libsnappyjava.jnilib
+Mac-x86_LIBNAME := libsnappyjava.dylib
Mac-x86_SNAPPY_FLAGS :=
Mac-x86_64_CXX := c++ -arch $(OS_ARCH)
Mac-x86_64_STRIP := strip -x
-Mac-x86_64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.5 -fvisibility=hidden
+Mac-x86_64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.7 -fvisibility=hidden -stdlib=libc++ -std=c++11
Mac-x86_64_LINKFLAGS := -dynamiclib
-Mac-x86_64_LIBNAME := libsnappyjava.jnilib
+Mac-x86_64_LIBNAME := libsnappyjava.dylib
Mac-x86_64_SNAPPY_FLAGS :=
+Mac-aarch64_CXX := c++ -arch arm64
+Mac-aarch64_STRIP := strip -x
+Mac-aarch64_CXXFLAGS := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.7 -fvisibility=hidden -stdlib=libc++ -std=c++11
+Mac-aarch64_LINKFLAGS := -dynamiclib
+Mac-aarch64_LIBNAME := libsnappyjava.dylib
+
FreeBSD-x86_64_CXX := $(CROSS_PREFIX)g++
FreeBSD-x86_64_STRIP := $(CROSS_PREFIX)strip
-FreeBSD-x86_64_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
+FreeBSD-x86_64_CXXFLAGS := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -std=c++11
FreeBSD-x86_64_LINKFLAGS := -shared -static-libgcc
FreeBSD-x86_64_LIBNAME := libsnappyjava.so
FreeBSD-x86_64_SNAPPY_FLAGS :=
Windows-x86_CXX := $(CROSS_PREFIX)g++
Windows-x86_STRIP := $(CROSS_PREFIX)strip
-Windows-x86_CXXFLAGS := -Ilib/inc_win -O2
+Windows-x86_CXXFLAGS := -Ilib/inc_win -O2 -std=c++11
Windows-x86_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-x86_LIBNAME := snappyjava.dll
Windows-x86_SNAPPY_FLAGS :=
Windows-x86_64_CXX := $(CROSS_PREFIX)g++
Windows-x86_64_STRIP := $(CROSS_PREFIX)strip
-Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2
+Windows-x86_64_CXXFLAGS := -Ilib/inc_win -O2 -std=c++11
Windows-x86_64_LINKFLAGS := -Wl,--kill-at -shared -static
Windows-x86_64_LIBNAME := snappyjava.dll
Windows-x86_64_SNAPPY_FLAGS :=
=====================================
Makefile.package
=====================================
@@ -28,8 +28,8 @@ DLL_DIR=src/main/resources/org/xerial/snappy/native
DLL_WIN=$(DLL_DIR)/Windows/x86/snappyjava.dll
DLL_WIN64=$(DLL_DIR)/Windows/amd64/snappyjava.dll
-DLL_MAC=$(DLL_DIR)/Mac/x86_64/libsnappyjava.jnilib
-DLL_TIGER_MAC=$(DLL_DIR)/Mac/i386/libsnappyjava.jnilib
+DLL_MAC=$(DLL_DIR)/Mac/x86_64/libsnappyjava.dylib
+DLL_TIGER_MAC=$(DLL_DIR)/Mac/i386/libsnappyjava.dylib
DLL_LINUX=$(DLL_DIR)/Linux/i386/libsnappyjava.so
DLL_AMD64=$(DLL_DIR)/Linux/amd64/libsnappyjava.so
=====================================
Milestone.md
=====================================
@@ -1,5 +1,27 @@
Since version 1.1.0.x, Java 6 (1.6) or higher is required.
+## snappy-java-1.1.8.3 (2021-01-20)
+ * Make pure-java Snappy thread-safe [#271](https://github.com/xerial/snappy-java/pull/271)
+ * Improved SnappyFramedInput/OutputStream performance by using java.util.zip.CRC32C [#269](https://github.com/xerial/snappy-java/pull/269)
+
+## snappy-java-1.1.8.2 (2020-11-28)
+ * Support Apple Silicon (M1, Mac-aarch64)
+ * Fixed the pure-java Snappy fallback logic when no native library for your platform is found.
+
+## snappy-java-1.1.8.1 (2020-11-09)
+ * Fixed an initialization issue when using a recent Mac OS X version [#265](https://github.com/xerial/snappy-java/pull/265)
+
+## snappy-java-1.1.8 (2020-10-20)
+ * Upgrade to [Snappy 1.1.8](https://github.com/google/snappy/releases/tag/1.1.8) with small performance improvements.
+
+## snappy-java-1.1.7.8 (2020-10-20)
+ * Big-endian support for pure-java Snappy implementation
+ * linux-aarch64 (arm64) binary embeds libstdc++ for portability
+ * internal: Fix make native-all target to support the latest version of dockcross
+
+## snappy-java-1.1.7.7 (2020-08-25)
+ * Built snappy-java with jdk8 to resolve #251 (java.lang.NoSuchMethodError)
+
## snappy-java-1.1.7.6 (2020-06-26)
* Added an experimental support of pure-java Snappy https://github.com/xerial/snappy-java#using-pure-java-snappy-implementation
* Pure-java snappy doesn't support Snappy.isValidCompressedBuffer methods, but the other methods, Snappy.compress, uncompress, SnappyInput/OutputStream, SnappyFramedInput/OutputStream, etc., should work as expected..
=====================================
README.md
=====================================
@@ -1,7 +1,6 @@
snappy-java [](https://travis-ci.org/xerial/snappy-java) [](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) [](http://javadoc-badge.appspot.com/org.xerial.snappy/snappy-java)
===
-snappy-java is a Java port of the snappy
-<http://code.google.com/p/snappy/>, a fast C++ compresser/decompresser developed by Google.
+snappy-java is a Java port of the [snappy](https://github.com/google/snappy), a fast C++ compresser/decompresser developed by Google.
## Features
* Fast compression/decompression around 200~400MB/sec.
@@ -9,8 +8,9 @@ snappy-java is a Java port of the snappy
* JNI-based implementation to achieve comparable performance to the native C++ version.
* Although snappy-java uses JNI, it can be used safely with multiple class loaders (e.g. Tomcat, etc.).
* Compression/decompression of Java primitive arrays (`float[]`, `double[]`, `int[]`, `short[]`, `long[]`, etc.)
- * To improve the compression ratios of these arrays, you can use a fast data-rearrangement implementation ([`BitShuffle`](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.3-M1/snappy-java-1.1.3-M1-javadoc.jar/!/org/xerial/snappy/BitShuffle.html)) before compression
- * Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux (64-bit). snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`).
+ * To improve the compression ratios of these arrays, you can use a fast data-rearrangement implementation ([`BitShuffle`](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/snappy/snappy-java/1.1.8/snappy-java-1.1.8-javadoc.jar/!/org/xerial/snappy/BitShuffle.html)) before compression
+ * Portable across various operating systems; Snappy-java contains native libraries built for Window/Mac/Linux, etc. snappy-java loads one of these libraries according to your machine environment (It looks system properties, `os.name` and `os.arch`).
+ * If no native library for your platform is found, snappy-java will fallback to [pure-java implementation](#using-pure-java-snappy-implementation).
* Simple usage. Add the snappy-java-(version).jar file to your classpath. Then call compression/decompression methods in `org.xerial.snappy.Snappy`.
* [Framing-format support](https://github.com/google/snappy/blob/master/framing_format.txt) (Since 1.1.0 version)
* OSGi support
=====================================
build.sbt
=====================================
@@ -19,7 +19,7 @@ developers := List(
scalaVersion in ThisBuild := "2.12.11"
-// For building jars for JDK7
+// For building jars for JDK8
javacOptions in ThisBuild ++= Seq("-source", "1.8", "-target", "1.8")
javacOptions in (Compile, compile) ++= Seq("-encoding", "UTF-8", "-Xlint:unchecked", "-Xlint:deprecation")
@@ -83,8 +83,8 @@ OsgiKeys.additionalHeaders := Map(
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x64",
"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=amd64",
"org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86",
- "org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86",
- "org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64",
+ "org/xerial/snappy/native/Mac/x86/libsnappyjava.dylib;osname=macosx;processor=x86",
+ "org/xerial/snappy/native/Mac/x86_64/libsnappyjava.dylib;osname=macosx;processor=x86-64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x64",
"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=amd64",
=====================================
docker/Makefile
=====================================
@@ -4,3 +4,6 @@ centos5-image:
# dockerhub login
# docker login --username=xerial
# docker push xerial/centos5-linux-x86_64:latest
+
+multiarch-crossbuild-image:
+ docker build https://github.com/iwasakims/crossbuild.git#fix-osxcross-cmake -t xerial/crossbuild:latest
=====================================
docker/dockcross-arm64
=====================================
@@ -0,0 +1,242 @@
+#!/usr/bin/env bash
+
+DEFAULT_DOCKCROSS_IMAGE=dockcross/linux-arm64
+
+#------------------------------------------------------------------------------
+# Helpers
+#
+err() {
+ echo -e >&2 ERROR: $@\\n
+}
+
+die() {
+ err $@
+ exit 1
+}
+
+has() {
+ # eg. has command update
+ local kind=$1
+ local name=$2
+
+ type -t $kind:$name | grep -q function
+}
+
+#------------------------------------------------------------------------------
+# Command handlers
+#
+command:update-image() {
+ docker pull $FINAL_IMAGE
+}
+
+help:update-image() {
+ echo Pull the latest $FINAL_IMAGE .
+}
+
+command:update-script() {
+ if cmp -s <( docker run --rm $FINAL_IMAGE ) $0; then
+ echo $0 is up to date
+ else
+ echo -n Updating $0 '... '
+ docker run --rm $FINAL_IMAGE > $0 && echo ok
+ fi
+}
+
+help:update-image() {
+ echo Update $0 from $FINAL_IMAGE .
+}
+
+command:update() {
+ command:update-image
+ command:update-script
+}
+
+help:update() {
+ echo Pull the latest $FINAL_IMAGE, and then update $0 from that.
+}
+
+command:help() {
+ if [[ $# != 0 ]]; then
+ if ! has command $1; then
+ err \"$1\" is not an dockcross command
+ command:help
+ elif ! has help $1; then
+ err No help found for \"$1\"
+ else
+ help:$1
+ fi
+ else
+ cat >&2 <<ENDHELP
+Usage: dockcross [options] [--] command [args]
+
+By default, run the given *command* in an dockcross Docker container.
+
+The *options* can be one of:
+
+ --args|-a Extra args to the *docker run* command
+ --image|-i Docker cross-compiler image to use
+ --config|-c Bash script to source before running this script
+
+
+Additionally, there are special update commands:
+
+ update-image
+ update-script
+ update
+
+For update command help use: $0 help <command>
+ENDHELP
+ exit 1
+ fi
+}
+
+#------------------------------------------------------------------------------
+# Option processing
+#
+special_update_command=''
+while [[ $# != 0 ]]; do
+ case $1 in
+
+ --)
+ shift
+ break
+ ;;
+
+ --args|-a)
+ ARG_ARGS="$2"
+ shift 2
+ ;;
+
+ --config|-c)
+ ARG_CONFIG="$2"
+ shift 2
+ ;;
+
+ --image|-i)
+ ARG_IMAGE="$2"
+ shift 2
+ ;;
+ update|update-image|update-script)
+ special_update_command=$1
+ break
+ ;;
+ -*)
+ err Unknown option \"$1\"
+ command:help
+ exit
+ ;;
+
+ *)
+ break
+ ;;
+
+ esac
+done
+
+# The precedence for options is:
+# 1. command-line arguments
+# 2. environment variables
+# 3. defaults
+
+# Source the config file if it exists
+DEFAULT_DOCKCROSS_CONFIG=~/.dockcross
+FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}}
+
+[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG"
+
+# Set the docker image
+FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}}
+
+# Handle special update command
+if [ "$special_update_command" != "" ]; then
+ case $special_update_command in
+
+ update)
+ command:update
+ exit $?
+ ;;
+
+ update-image)
+ command:update-image
+ exit $?
+ ;;
+
+ update-script)
+ command:update-script
+ exit $?
+ ;;
+
+ esac
+fi
+
+# Set the docker run extra args (if any)
+FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}}
+
+# Bash on Ubuntu on Windows
+UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "")
+# MSYS, Git Bash, etc.
+MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "")
+
+if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" ]; then
+ USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )")
+fi
+
+# Change the PWD when working in Docker on Windows
+if [ -n "$UBUNTU_ON_WINDOWS" ]; then
+ WSL_ROOT="/mnt/"
+ CFG_FILE=/etc/wsl.conf
+ if [ -f "$CFG_FILE" ]; then
+ CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g')
+ eval "$CFG_CONTENT"
+ if [ -n "$root" ]; then
+ WSL_ROOT=$root
+ fi
+ fi
+ HOST_PWD=`pwd -P`
+ HOST_PWD=${HOST_PWD/$WSL_ROOT//}
+elif [ -n "$MSYS" ]; then
+ HOST_PWD=$PWD
+ HOST_PWD=${HOST_PWD/\//}
+ HOST_PWD=${HOST_PWD/\//:\/}
+else
+ HOST_PWD=$PWD
+fi
+
+# Mount Additional Volumes
+if [ -z "$SSH_DIR" ]; then
+ SSH_DIR="$HOME/.ssh"
+fi
+
+HOST_VOLUMES=
+if [ -e "$SSH_DIR" ]; then
+ HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh"
+fi
+
+#------------------------------------------------------------------------------
+# Now, finally, run the command in a container
+#
+tty -s && TTY_ARGS=-ti || TTY_ARGS=
+CONTAINER_NAME=dockcross_$RANDOM
+docker run $TTY_ARGS --name $CONTAINER_NAME \
+ -v "$HOST_PWD":/work \
+ $HOST_VOLUMES \
+ "${USER_IDS[@]}" \
+ $FINAL_ARGS \
+ $FINAL_IMAGE "$@"
+run_exit_code=$?
+
+exit $run_exit_code
+
+################################################################################
+#
+# This image is not intended to be run manually.
+#
+# To create a dockcross helper script for the
+# dockcross/linux-arm64 image, run:
+#
+# docker run --rm dockcross/linux-arm64 > dockcross-linux-arm64
+# chmod +x dockcross-linux-arm64
+#
+# You may then wish to move the dockcross script to your PATH.
+#
+################################################################################
=====================================
src/main/java/org/xerial/snappy/PureJavaCrc32C.java
=====================================
@@ -48,8 +48,7 @@ public class PureJavaCrc32C
/** {@inheritDoc} */
public long getValue()
{
- long ret = crc;
- return (~ret) & 0xffffffffL;
+ return (~crc) & 0xffffffffL;
}
/** {@inheritDoc} */
=====================================
src/main/java/org/xerial/snappy/SnappyFramed.java
=====================================
@@ -1,125 +1,165 @@
-/*
- * Created: Apr 12, 2013
- */
-package org.xerial.snappy;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ReadableByteChannel;
-
-/**
- * Constants and utilities for implementing x-snappy-framed.
- *
- * @author Brett Okken
- * @since 1.1.0
- */
-final class SnappyFramed
-{
- public static final int COMPRESSED_DATA_FLAG = 0x00;
-
- public static final int UNCOMPRESSED_DATA_FLAG = 0x01;
-
- public static final int STREAM_IDENTIFIER_FLAG = 0xff;
-
- private static final int MASK_DELTA = 0xa282ead8;
-
- /**
- * The header consists of the stream identifier flag, 3 bytes indicating a
- * length of 6, and "sNaPpY" in ASCII.
- */
- public static final byte[] HEADER_BYTES = new byte[] {
- (byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61,
- 0x50, 0x70, 0x59};
-
- public static int maskedCrc32c(byte[] data)
- {
- return maskedCrc32c(data, 0, data.length);
- }
-
- public static int maskedCrc32c(byte[] data, int offset, int length)
- {
- final PureJavaCrc32C crc32c = new PureJavaCrc32C();
- crc32c.update(data, offset, length);
- return mask(crc32c.getIntegerValue());
- }
-
- /**
- * Checksums are not stored directly, but masked, as checksumming data and
- * then its own checksum can be problematic. The masking is the same as used
- * in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant
- * 0xa282ead8 (using wraparound as normal for unsigned integers). This is
- * equivalent to the following C code:
- * <p/>
- * <pre>
- * uint32_t mask_checksum(uint32_t x) {
- * return ((x >> 15) | (x << 17)) + 0xa282ead8;
- * }
- * </pre>
- */
- public static int mask(int crc)
- {
- // Rotate right by 15 bits and add a constant.
- return ((crc >>> 15) | (crc << 17)) + MASK_DELTA;
- }
-
- static final int readBytes(ReadableByteChannel source, ByteBuffer dest)
- throws IOException
- {
- // tells how many bytes to read.
- final int expectedLength = dest.remaining();
-
- int totalRead = 0;
-
- // how many bytes were read.
- int lastRead = source.read(dest);
-
- totalRead = lastRead;
-
- // if we did not read as many bytes as we had hoped, try reading again.
- if (lastRead < expectedLength) {
- // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading.
- while (dest.remaining() != 0 && lastRead != -1) {
- lastRead = source.read(dest);
-
- // if we got EOF, do not add to total read.
- if (lastRead != -1) {
- totalRead += lastRead;
- }
- }
- }
-
- if (totalRead > 0) {
- dest.limit(dest.position());
- }
- else {
- dest.position(dest.limit());
- }
-
- return totalRead;
- }
-
- static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer)
- throws IOException
- {
- if (skip <= 0) {
- return 0;
- }
-
- int toSkip = skip;
- int skipped = 0;
- while (toSkip > 0 && skipped != -1) {
- buffer.clear();
- if (toSkip < buffer.capacity()) {
- buffer.limit(toSkip);
- }
-
- skipped = source.read(buffer);
- if (skipped > 0) {
- toSkip -= skipped;
- }
- }
-
- buffer.clear();
- return skip - toSkip;
- }
-}
+/*
+ * Created: Apr 12, 2013
+ */
+package org.xerial.snappy;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.Checksum;
+
+/**
+ * Constants and utilities for implementing x-snappy-framed.
+ *
+ * @author Brett Okken
+ * @since 1.1.0
+ */
+final class SnappyFramed
+{
+ public static final int COMPRESSED_DATA_FLAG = 0x00;
+
+ public static final int UNCOMPRESSED_DATA_FLAG = 0x01;
+
+ public static final int STREAM_IDENTIFIER_FLAG = 0xff;
+
+ private static final int MASK_DELTA = 0xa282ead8;
+
+ private static final Supplier<Checksum> CHECKSUM_SUPPLIER;
+
+ static
+ {
+ Supplier<Checksum> supplier = null;
+ try
+ {
+ final Class crc32cClazz = Class.forName("java.util.zip.CRC32C");
+ final MethodHandles.Lookup lookup = MethodHandles.publicLookup();
+
+ final MethodHandle conHandle = lookup.findConstructor(crc32cClazz, MethodType.methodType(void.class))
+ .asType(MethodType.methodType(Checksum.class));
+ supplier = () -> {
+ try
+ {
+ return (Checksum) conHandle.invokeExact();
+ }
+ catch (Throwable e)
+ {
+ throw new IllegalStateException(e);
+ }
+ };
+ }
+ catch(Throwable t)
+ {
+ Logger.getLogger(SnappyFramed.class.getName())
+ .log(Level.FINE, "java.util.zip.CRC32C not loaded, using PureJavaCrc32C", t);
+ supplier = null;
+ }
+
+ CHECKSUM_SUPPLIER = supplier != null ? supplier : PureJavaCrc32C::new;
+ }
+
+ /**
+ * The header consists of the stream identifier flag, 3 bytes indicating a
+ * length of 6, and "sNaPpY" in ASCII.
+ */
+ public static final byte[] HEADER_BYTES = new byte[] {
+ (byte) STREAM_IDENTIFIER_FLAG, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61,
+ 0x50, 0x70, 0x59};
+
+ public static Checksum getCRC32C()
+ {
+ return CHECKSUM_SUPPLIER.get();
+ }
+
+ public static int maskedCrc32c(Checksum crc32c, byte[] data, int offset, int length)
+ {
+ crc32c.reset();
+ crc32c.update(data, offset, length);
+ return mask((int) crc32c.getValue());
+ }
+
+ /**
+ * Checksums are not stored directly, but masked, as checksumming data and
+ * then its own checksum can be problematic. The masking is the same as used
+ * in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant
+ * 0xa282ead8 (using wraparound as normal for unsigned integers). This is
+ * equivalent to the following C code:
+ * <p/>
+ * <pre>
+ * uint32_t mask_checksum(uint32_t x) {
+ * return ((x >> 15) | (x << 17)) + 0xa282ead8;
+ * }
+ * </pre>
+ */
+ public static int mask(int crc)
+ {
+ // Rotate right by 15 bits and add a constant.
+ return ((crc >>> 15) | (crc << 17)) + MASK_DELTA;
+ }
+
+ static final int readBytes(ReadableByteChannel source, ByteBuffer dest)
+ throws IOException
+ {
+ // tells how many bytes to read.
+ final int expectedLength = dest.remaining();
+
+ int totalRead = 0;
+
+ // how many bytes were read.
+ int lastRead = source.read(dest);
+
+ totalRead = lastRead;
+
+ // if we did not read as many bytes as we had hoped, try reading again.
+ if (lastRead < expectedLength) {
+ // as long the buffer is not full (remaining() == 0) and we have not reached EOF (lastRead == -1) keep reading.
+ while (dest.remaining() != 0 && lastRead != -1) {
+ lastRead = source.read(dest);
+
+ // if we got EOF, do not add to total read.
+ if (lastRead != -1) {
+ totalRead += lastRead;
+ }
+ }
+ }
+
+ if (totalRead > 0) {
+ dest.limit(dest.position());
+ }
+ else {
+ dest.position(dest.limit());
+ }
+
+ return totalRead;
+ }
+
+ static int skip(final ReadableByteChannel source, final int skip, final ByteBuffer buffer)
+ throws IOException
+ {
+ if (skip <= 0) {
+ return 0;
+ }
+
+ int toSkip = skip;
+ int skipped = 0;
+ while (toSkip > 0 && skipped != -1) {
+ buffer.clear();
+ if (toSkip < buffer.capacity()) {
+ buffer.limit(toSkip);
+ }
+
+ skipped = source.read(buffer);
+ if (skipped > 0) {
+ toSkip -= skipped;
+ }
+ }
+
+ buffer.clear();
+ return skip - toSkip;
+ }
+}
=====================================
src/main/java/org/xerial/snappy/SnappyFramedInputStream.java
=====================================
@@ -1,683 +1,685 @@
-/*
- * Created: Apr 15, 2013
- */
-package org.xerial.snappy;
-
-import static java.lang.Math.min;
-import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
-import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
-import static org.xerial.snappy.SnappyFramed.STREAM_IDENTIFIER_FLAG;
-import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
-import static org.xerial.snappy.SnappyFramed.readBytes;
-import static org.xerial.snappy.SnappyFramedOutputStream.MAX_BLOCK_SIZE;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.ClosedChannelException;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-import java.util.Arrays;
-
-import org.xerial.snappy.pool.BufferPool;
-import org.xerial.snappy.pool.DefaultPoolFactory;
-
-/**
- * Implements the <a
- * href="http://snappy.googlecode.com/svn/trunk/framing_format.txt"
- * >x-snappy-framed</a> as an {@link InputStream} and
- * {@link ReadableByteChannel}.
- *
- * @author Brett Okken
- * @since 1.1.0
- */
-public final class SnappyFramedInputStream
- extends InputStream
- implements
- ReadableByteChannel
-{
-
- private final ReadableByteChannel rbc;
- private final ByteBuffer frameHeader;
- private final boolean verifyChecksums;
- private final BufferPool bufferPool;
-
- /**
- * A single frame read from the underlying {@link InputStream}.
- */
- private ByteBuffer input;
-
- /**
- * The decompressed data from {@link #input}.
- */
- private ByteBuffer uncompressedDirect;
-
- /**
- * Indicates if this instance has been closed.
- */
- private boolean closed;
-
- /**
- * Indicates if we have reached the EOF on {@link #input}.
- */
- private boolean eof;
-
- /**
- * The position in {@link #input} buffer to read to.
- */
- private int valid;
-
- /**
- * The next position to read from {@link #buffer}.
- */
- private int position;
-
- /**
- * Buffer contains a copy of the uncompressed data for the block.
- */
- private byte[] buffer;
-
- /**
- * Creates a Snappy input stream to read data from the specified underlying
- * input stream.
- * <p>
- * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
- * </p>
- *
- * @param in the underlying input stream. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedInputStream(InputStream in)
- throws IOException
- {
- this(in, true, DefaultPoolFactory.getDefaultPool());
- }
-
- /**
- * Creates a Snappy input stream to read data from the specified underlying
- * input stream.
- *
- * @param in the underlying input stream. Must not be {@code null}.
- * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedInputStream(InputStream in, BufferPool bufferPool)
- throws IOException
- {
- this(in, true, bufferPool);
- }
-
- /**
- * Creates a Snappy input stream to read data from the specified underlying
- * input stream.
- * <p>
- * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
- * </p>
- *
- * @param in the underlying input stream. Must not be {@code null}.
- * @param verifyChecksums if true, checksums in input stream will be verified
- * @throws IOException
- */
- public SnappyFramedInputStream(InputStream in, boolean verifyChecksums)
- throws IOException
- {
- this(in, verifyChecksums, DefaultPoolFactory.getDefaultPool());
- }
-
- /**
- * Creates a Snappy input stream to read data from the specified underlying
- * input stream.
- *
- * @param in the underlying input stream. Must not be {@code null}.
- * @param verifyChecksums if true, checksums in input stream will be verified
- * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedInputStream(InputStream in, boolean verifyChecksums,
- BufferPool bufferPool)
- throws IOException
- {
- this(Channels.newChannel(in), verifyChecksums, bufferPool);
- }
-
- /**
- * Creates a Snappy input stream to read data from the specified underlying
- * channel.
- *
- * @param in the underlying readable channel. Must not be {@code null}.
- * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedInputStream(ReadableByteChannel in, BufferPool bufferPool)
- throws IOException
- {
- this(in, true, bufferPool);
- }
-
- /**
- * Creates a Snappy input stream to read data from the specified underlying
- * channel.
- * <p>
- * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
- * </p>
- *
- * @param in the underlying readable channel. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedInputStream(ReadableByteChannel in)
- throws IOException
- {
- this(in, true);
- }
-
- /**
- * Creates a Snappy input stream to read data from the specified underlying
- * channel.
- * <p>
- * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
- * </p>
- *
- * @param in the underlying readable channel. Must not be {@code null}.
- * @param verifyChecksums if true, checksums in input stream will be verified
- * @throws IOException
- */
- public SnappyFramedInputStream(ReadableByteChannel in,
- boolean verifyChecksums)
- throws IOException
- {
- this(in, verifyChecksums, DefaultPoolFactory.getDefaultPool());
- }
-
- /**
- * Creates a Snappy input stream to read data from the specified underlying
- * channel.
- *
- * @param in the underlying readable channel. Must not be {@code null}.
- * @param verifyChecksums if true, checksums in input stream will be verified
- * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedInputStream(ReadableByteChannel in,
- boolean verifyChecksums, BufferPool bufferPool)
- throws IOException
- {
- if (in == null) {
- throw new NullPointerException("in is null");
- }
-
- if (bufferPool == null) {
- throw new NullPointerException("bufferPool is null");
- }
-
- this.bufferPool = bufferPool;
- this.rbc = in;
- this.verifyChecksums = verifyChecksums;
-
- allocateBuffersBasedOnSize(MAX_BLOCK_SIZE + 5);
- this.frameHeader = ByteBuffer.allocate(4);
-
- // stream must begin with stream header
- final byte[] expectedHeader = HEADER_BYTES;
- final byte[] actualHeader = new byte[expectedHeader.length];
- final ByteBuffer actualBuffer = ByteBuffer.wrap(actualHeader);
-
- final int read = SnappyFramed.readBytes(in, actualBuffer);
- if (read < expectedHeader.length) {
- throw new EOFException(
- "encountered EOF while reading stream header");
- }
- if (!Arrays.equals(expectedHeader, actualHeader)) {
- throw new IOException("invalid stream header");
- }
- }
-
- /**
- * @param size
- */
- private void allocateBuffersBasedOnSize(int size)
- {
- if (input != null) {
- bufferPool.releaseDirect(input);
- }
-
- if (uncompressedDirect != null) {
- bufferPool.releaseDirect(uncompressedDirect);
- }
-
- if (buffer != null) {
- bufferPool.releaseArray(buffer);
- }
-
- input = bufferPool.allocateDirect(size);
- final int maxCompressedLength = Snappy.maxCompressedLength(size);
- uncompressedDirect = bufferPool.allocateDirect(maxCompressedLength);
- buffer = bufferPool.allocateArray(maxCompressedLength);
- }
-
- @Override
- public int read()
- throws IOException
- {
- if (closed) {
- return -1;
- }
- if (!ensureBuffer()) {
- return -1;
- }
- return buffer[position++] & 0xFF;
- }
-
- @Override
- public int read(byte[] output, int offset, int length)
- throws IOException
- {
-
- if (output == null) {
- throw new IllegalArgumentException("output is null");
- }
-
- if (offset < 0 || length < 0 || offset + length > output.length) {
- throw new IllegalArgumentException("invalid offset [" + offset
- + "] and length [" + length + ']');
- }
-
- if (closed) {
- throw new ClosedChannelException();
- }
-
- if (length == 0) {
- return 0;
- }
- if (!ensureBuffer()) {
- return -1;
- }
-
- final int size = min(length, available());
- System.arraycopy(buffer, position, output, offset, size);
- position += size;
- return size;
- }
-
- @Override
- public int available()
- throws IOException
- {
- if (closed) {
- return 0;
- }
- return valid - position;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isOpen()
- {
- return !closed;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public int read(ByteBuffer dst)
- throws IOException
- {
-
- if (dst == null) {
- throw new IllegalArgumentException("dst is null");
- }
-
- if (closed) {
- throw new ClosedChannelException();
- }
-
- if (dst.remaining() == 0) {
- return 0;
- }
- if (!ensureBuffer()) {
- return -1;
- }
-
- final int size = min(dst.remaining(), available());
- dst.put(buffer, position, size);
- position += size;
- return size;
- }
-
- /**
- * Transfers the entire content of this {@link InputStream} to <i>os</i>.
- * This potentially limits the amount of buffering required to decompress
- * content.
- * <p>
- * Unlike {@link #read(byte[], int, int)}, this method does not need to be
- * called multiple times. A single call will transfer all available content.
- * Any calls after the source has been exhausted will result in a return
- * value of {@code 0}.
- * </p>
- *
- * @param os The destination to write decompressed content to.
- * @return The number of bytes transferred.
- * @throws IOException
- * @since 1.1.1
- */
- public long transferTo(OutputStream os)
- throws IOException
- {
- if (os == null) {
- throw new IllegalArgumentException("os is null");
- }
-
- if (closed) {
- throw new ClosedChannelException();
- }
-
- long totTransfered = 0;
-
- while (ensureBuffer()) {
- final int available = available();
- os.write(buffer, position, available);
- position += available;
- totTransfered += available;
- }
-
- return totTransfered;
- }
-
- /**
- * Transfers the entire content of this {@link ReadableByteChannel} to
- * <i>wbc</i>. This potentially limits the amount of buffering required to
- * decompress content.
- * <p/>
- * <p>
- * Unlike {@link #read(ByteBuffer)}, this method does not need to be called
- * multiple times. A single call will transfer all available content. Any
- * calls after the source has been exhausted will result in a return value
- * of {@code 0}.
- * </p>
- *
- * @param wbc The destination to write decompressed content to.
- * @return The number of bytes transferred.
- * @throws IOException
- * @since 1.1.1
- */
- public long transferTo(WritableByteChannel wbc)
- throws IOException
- {
- if (wbc == null) {
- throw new IllegalArgumentException("wbc is null");
- }
-
- if (closed) {
- throw new ClosedChannelException();
- }
-
- final ByteBuffer bb = ByteBuffer.wrap(buffer);
-
- long totTransfered = 0;
-
- while (ensureBuffer()) {
- bb.clear();
- bb.position(position);
- bb.limit(position + available());
-
- wbc.write(bb);
-
- final int written = bb.position() - position;
- position += written;
-
- totTransfered += written;
- }
-
- return totTransfered;
- }
-
- @Override
- public void close()
- throws IOException
- {
- try {
- rbc.close();
- }
- finally {
- if (!closed) {
- closed = true;
-
- if (input != null) {
- bufferPool.releaseDirect(input);
- input = null;
- }
-
- if (uncompressedDirect != null) {
- bufferPool.releaseDirect(uncompressedDirect);
- uncompressedDirect = null;
- }
-
- if (buffer != null) {
- bufferPool.releaseArray(buffer);
- buffer = null;
- }
- }
- }
- }
-
- static enum FrameAction
- {
- RAW, SKIP, UNCOMPRESS;
- }
-
- public static final class FrameMetaData
- {
- final int length;
- final FrameAction frameAction;
-
- /**
- * @param frameAction
- * @param length
- */
- public FrameMetaData(FrameAction frameAction, int length)
- {
- super();
- this.frameAction = frameAction;
- this.length = length;
- }
- }
-
- public static final class FrameData
- {
- final int checkSum;
- final int offset;
-
- /**
- * @param checkSum
- * @param offset
- */
- public FrameData(int checkSum, int offset)
- {
- super();
- this.checkSum = checkSum;
- this.offset = offset;
- }
- }
-
- private boolean ensureBuffer()
- throws IOException
- {
- if (available() > 0) {
- return true;
- }
- if (eof) {
- return false;
- }
-
- if (!readBlockHeader()) {
- eof = true;
- return false;
- }
-
- // get action based on header
- final FrameMetaData frameMetaData = getFrameMetaData(frameHeader);
-
- if (FrameAction.SKIP == frameMetaData.frameAction) {
- SnappyFramed.skip(rbc, frameMetaData.length,
- ByteBuffer.wrap(buffer));
- return ensureBuffer();
- }
-
- if (frameMetaData.length > input.capacity()) {
- allocateBuffersBasedOnSize(frameMetaData.length);
- }
-
- input.clear();
- input.limit(frameMetaData.length);
-
- final int actualRead = readBytes(rbc, input);
- if (actualRead != frameMetaData.length) {
- throw new EOFException("unexpectd EOF when reading frame");
- }
- input.flip();
-
- final FrameData frameData = getFrameData(input);
-
- if (FrameAction.UNCOMPRESS == frameMetaData.frameAction) {
-
- input.position(frameData.offset);
-
- final int uncompressedLength = Snappy.uncompressedLength(input);
-
- if (uncompressedLength > uncompressedDirect.capacity()) {
- bufferPool.releaseDirect(uncompressedDirect);
- bufferPool.releaseArray(buffer);
- uncompressedDirect = bufferPool.allocateDirect(uncompressedLength);
- buffer = bufferPool.allocateArray(uncompressedLength);
- }
-
- uncompressedDirect.clear();
-
- this.valid = Snappy.uncompress(input, uncompressedDirect);
-
- uncompressedDirect.get(buffer, 0, valid);
- this.position = 0;
- }
- else {
- // we need to start reading at the offset
- input.position(frameData.offset);
- this.position = 0;
- this.valid = input.remaining();
- this.input.get(buffer, 0, input.remaining());
- }
-
- if (verifyChecksums) {
- final int actualCrc32c = SnappyFramed.maskedCrc32c(buffer,
- position, valid - position);
- if (frameData.checkSum != actualCrc32c) {
- throw new IOException("Corrupt input: invalid checksum");
- }
- }
-
- return true;
- }
-
- private boolean readBlockHeader()
- throws IOException
- {
- frameHeader.clear();
- int read = readBytes(rbc, frameHeader);
-
- if (read == -1) {
- return false;
- }
-
- if (read < frameHeader.capacity()) {
- throw new EOFException("encountered EOF while reading block header");
- }
- frameHeader.flip();
-
- return true;
- }
-
- /**
- * @param frameHeader
- * @return
- * @throws IOException
- */
- private FrameMetaData getFrameMetaData(ByteBuffer frameHeader)
- throws IOException
- {
-
- assert frameHeader.hasArray();
-
- final byte[] frameHeaderArray = frameHeader.array();
-
- int length = (frameHeaderArray[1] & 0xFF);
- length |= (frameHeaderArray[2] & 0xFF) << 8;
- length |= (frameHeaderArray[3] & 0xFF) << 16;
-
- int minLength = 0;
- final FrameAction frameAction;
- final int flag = frameHeaderArray[0] & 0xFF;
- switch (flag) {
- case COMPRESSED_DATA_FLAG:
- frameAction = FrameAction.UNCOMPRESS;
- minLength = 5;
- break;
- case UNCOMPRESSED_DATA_FLAG:
- frameAction = FrameAction.RAW;
- minLength = 5;
- break;
- case STREAM_IDENTIFIER_FLAG:
- if (length != 6) {
- throw new IOException(
- "stream identifier chunk with invalid length: "
- + length);
- }
- frameAction = FrameAction.SKIP;
- minLength = 6;
- break;
- default:
- // Reserved unskippable chunks (chunk types 0x02-0x7f)
- if (flag <= 0x7f) {
- throw new IOException("unsupported unskippable chunk: "
- + Integer.toHexString(flag));
- }
-
- // all that is left is Reserved skippable chunks (chunk types
- // 0x80-0xfe)
- frameAction = FrameAction.SKIP;
- minLength = 0;
- }
-
- if (length < minLength) {
- throw new IOException("invalid length: " + length
- + " for chunk flag: " + Integer.toHexString(flag));
- }
-
- return new FrameMetaData(frameAction, length);
- }
-
- /**
- * @param content
- * @return
- * @throws IOException
- */
- private FrameData getFrameData(ByteBuffer content)
- throws IOException
- {
- return new FrameData(getCrc32c(content), 4);
- }
-
- private int getCrc32c(ByteBuffer content)
- {
-
- final int position = content.position();
-
- return ((content.get(position + 3) & 0xFF) << 24)
- | ((content.get(position + 2) & 0xFF) << 16)
- | ((content.get(position + 1) & 0xFF) << 8)
- | (content.get(position) & 0xFF);
- }
-}
+/*
+ * Created: Apr 15, 2013
+ */
+package org.xerial.snappy;
+
+import static java.lang.Math.min;
+import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
+import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
+import static org.xerial.snappy.SnappyFramed.STREAM_IDENTIFIER_FLAG;
+import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
+import static org.xerial.snappy.SnappyFramed.readBytes;
+import static org.xerial.snappy.SnappyFramedOutputStream.MAX_BLOCK_SIZE;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.util.Arrays;
+import java.util.zip.Checksum;
+
+import org.xerial.snappy.pool.BufferPool;
+import org.xerial.snappy.pool.DefaultPoolFactory;
+
+/**
+ * Implements the <a
+ * href="http://snappy.googlecode.com/svn/trunk/framing_format.txt"
+ * >x-snappy-framed</a> as an {@link InputStream} and
+ * {@link ReadableByteChannel}.
+ *
+ * @author Brett Okken
+ * @since 1.1.0
+ */
+public final class SnappyFramedInputStream
+ extends InputStream
+ implements
+ ReadableByteChannel
+{
+
+ private final Checksum crc32 = SnappyFramed.getCRC32C();
+ private final ReadableByteChannel rbc;
+ private final ByteBuffer frameHeader;
+ private final boolean verifyChecksums;
+ private final BufferPool bufferPool;
+
+ /**
+ * A single frame read from the underlying {@link InputStream}.
+ */
+ private ByteBuffer input;
+
+ /**
+ * The decompressed data from {@link #input}.
+ */
+ private ByteBuffer uncompressedDirect;
+
+ /**
+ * Indicates if this instance has been closed.
+ */
+ private boolean closed;
+
+ /**
+ * Indicates if we have reached the EOF on {@link #input}.
+ */
+ private boolean eof;
+
+ /**
+ * The position in {@link #input} buffer to read to.
+ */
+ private int valid;
+
+ /**
+ * The next position to read from {@link #buffer}.
+ */
+ private int position;
+
+ /**
+ * Buffer contains a copy of the uncompressed data for the block.
+ */
+ private byte[] buffer;
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * input stream.
+ * <p>
+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
+ * </p>
+ *
+ * @param in the underlying input stream. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedInputStream(InputStream in)
+ throws IOException
+ {
+ this(in, true, DefaultPoolFactory.getDefaultPool());
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * input stream.
+ *
+ * @param in the underlying input stream. Must not be {@code null}.
+ * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedInputStream(InputStream in, BufferPool bufferPool)
+ throws IOException
+ {
+ this(in, true, bufferPool);
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * input stream.
+ * <p>
+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
+ * </p>
+ *
+ * @param in the underlying input stream. Must not be {@code null}.
+ * @param verifyChecksums if true, checksums in input stream will be verified
+ * @throws IOException
+ */
+ public SnappyFramedInputStream(InputStream in, boolean verifyChecksums)
+ throws IOException
+ {
+ this(in, verifyChecksums, DefaultPoolFactory.getDefaultPool());
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * input stream.
+ *
+ * @param in the underlying input stream. Must not be {@code null}.
+ * @param verifyChecksums if true, checksums in input stream will be verified
+ * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedInputStream(InputStream in, boolean verifyChecksums,
+ BufferPool bufferPool)
+ throws IOException
+ {
+ this(Channels.newChannel(in), verifyChecksums, bufferPool);
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * channel.
+ *
+ * @param in the underlying readable channel. Must not be {@code null}.
+ * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedInputStream(ReadableByteChannel in, BufferPool bufferPool)
+ throws IOException
+ {
+ this(in, true, bufferPool);
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * channel.
+ * <p>
+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
+ * </p>
+ *
+ * @param in the underlying readable channel. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedInputStream(ReadableByteChannel in)
+ throws IOException
+ {
+ this(in, true);
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * channel.
+ * <p>
+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
+ * </p>
+ *
+ * @param in the underlying readable channel. Must not be {@code null}.
+ * @param verifyChecksums if true, checksums in input stream will be verified
+ * @throws IOException
+ */
+ public SnappyFramedInputStream(ReadableByteChannel in,
+ boolean verifyChecksums)
+ throws IOException
+ {
+ this(in, verifyChecksums, DefaultPoolFactory.getDefaultPool());
+ }
+
+ /**
+ * Creates a Snappy input stream to read data from the specified underlying
+ * channel.
+ *
+ * @param in the underlying readable channel. Must not be {@code null}.
+ * @param verifyChecksums if true, checksums in input stream will be verified
+ * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedInputStream(ReadableByteChannel in,
+ boolean verifyChecksums, BufferPool bufferPool)
+ throws IOException
+ {
+ if (in == null) {
+ throw new NullPointerException("in is null");
+ }
+
+ if (bufferPool == null) {
+ throw new NullPointerException("bufferPool is null");
+ }
+
+ this.bufferPool = bufferPool;
+ this.rbc = in;
+ this.verifyChecksums = verifyChecksums;
+
+ allocateBuffersBasedOnSize(MAX_BLOCK_SIZE + 5);
+ this.frameHeader = ByteBuffer.allocate(4);
+
+ // stream must begin with stream header
+ final byte[] expectedHeader = HEADER_BYTES;
+ final byte[] actualHeader = new byte[expectedHeader.length];
+ final ByteBuffer actualBuffer = ByteBuffer.wrap(actualHeader);
+
+ final int read = SnappyFramed.readBytes(in, actualBuffer);
+ if (read < expectedHeader.length) {
+ throw new EOFException(
+ "encountered EOF while reading stream header");
+ }
+ if (!Arrays.equals(expectedHeader, actualHeader)) {
+ throw new IOException("invalid stream header");
+ }
+ }
+
+ /**
+ * @param size
+ */
+ private void allocateBuffersBasedOnSize(int size)
+ {
+ if (input != null) {
+ bufferPool.releaseDirect(input);
+ }
+
+ if (uncompressedDirect != null) {
+ bufferPool.releaseDirect(uncompressedDirect);
+ }
+
+ if (buffer != null) {
+ bufferPool.releaseArray(buffer);
+ }
+
+ input = bufferPool.allocateDirect(size);
+ final int maxCompressedLength = Snappy.maxCompressedLength(size);
+ uncompressedDirect = bufferPool.allocateDirect(maxCompressedLength);
+ buffer = bufferPool.allocateArray(maxCompressedLength);
+ }
+
+ @Override
+ public int read()
+ throws IOException
+ {
+ if (closed) {
+ return -1;
+ }
+ if (!ensureBuffer()) {
+ return -1;
+ }
+ return buffer[position++] & 0xFF;
+ }
+
+ @Override
+ public int read(byte[] output, int offset, int length)
+ throws IOException
+ {
+
+ if (output == null) {
+ throw new IllegalArgumentException("output is null");
+ }
+
+ if (offset < 0 || length < 0 || offset + length > output.length) {
+ throw new IllegalArgumentException("invalid offset [" + offset
+ + "] and length [" + length + ']');
+ }
+
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ if (length == 0) {
+ return 0;
+ }
+ if (!ensureBuffer()) {
+ return -1;
+ }
+
+ final int size = min(length, available());
+ System.arraycopy(buffer, position, output, offset, size);
+ position += size;
+ return size;
+ }
+
+ @Override
+ public int available()
+ throws IOException
+ {
+ if (closed) {
+ return 0;
+ }
+ return valid - position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isOpen()
+ {
+ return !closed;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int read(ByteBuffer dst)
+ throws IOException
+ {
+
+ if (dst == null) {
+ throw new IllegalArgumentException("dst is null");
+ }
+
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ if (dst.remaining() == 0) {
+ return 0;
+ }
+ if (!ensureBuffer()) {
+ return -1;
+ }
+
+ final int size = min(dst.remaining(), available());
+ dst.put(buffer, position, size);
+ position += size;
+ return size;
+ }
+
+ /**
+ * Transfers the entire content of this {@link InputStream} to <i>os</i>.
+ * This potentially limits the amount of buffering required to decompress
+ * content.
+ * <p>
+ * Unlike {@link #read(byte[], int, int)}, this method does not need to be
+ * called multiple times. A single call will transfer all available content.
+ * Any calls after the source has been exhausted will result in a return
+ * value of {@code 0}.
+ * </p>
+ *
+ * @param os The destination to write decompressed content to.
+ * @return The number of bytes transferred.
+ * @throws IOException
+ * @since 1.1.1
+ */
+ public long transferTo(OutputStream os)
+ throws IOException
+ {
+ if (os == null) {
+ throw new IllegalArgumentException("os is null");
+ }
+
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ long totTransfered = 0;
+
+ while (ensureBuffer()) {
+ final int available = available();
+ os.write(buffer, position, available);
+ position += available;
+ totTransfered += available;
+ }
+
+ return totTransfered;
+ }
+
+ /**
+ * Transfers the entire content of this {@link ReadableByteChannel} to
+ * <i>wbc</i>. This potentially limits the amount of buffering required to
+ * decompress content.
+ * <p/>
+ * <p>
+ * Unlike {@link #read(ByteBuffer)}, this method does not need to be called
+ * multiple times. A single call will transfer all available content. Any
+ * calls after the source has been exhausted will result in a return value
+ * of {@code 0}.
+ * </p>
+ *
+ * @param wbc The destination to write decompressed content to.
+ * @return The number of bytes transferred.
+ * @throws IOException
+ * @since 1.1.1
+ */
+ public long transferTo(WritableByteChannel wbc)
+ throws IOException
+ {
+ if (wbc == null) {
+ throw new IllegalArgumentException("wbc is null");
+ }
+
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ final ByteBuffer bb = ByteBuffer.wrap(buffer);
+
+ long totTransfered = 0;
+
+ while (ensureBuffer()) {
+ bb.clear();
+ bb.position(position);
+ bb.limit(position + available());
+
+ wbc.write(bb);
+
+ final int written = bb.position() - position;
+ position += written;
+
+ totTransfered += written;
+ }
+
+ return totTransfered;
+ }
+
+ @Override
+ public void close()
+ throws IOException
+ {
+ try {
+ rbc.close();
+ }
+ finally {
+ if (!closed) {
+ closed = true;
+
+ if (input != null) {
+ bufferPool.releaseDirect(input);
+ input = null;
+ }
+
+ if (uncompressedDirect != null) {
+ bufferPool.releaseDirect(uncompressedDirect);
+ uncompressedDirect = null;
+ }
+
+ if (buffer != null) {
+ bufferPool.releaseArray(buffer);
+ buffer = null;
+ }
+ }
+ }
+ }
+
+ static enum FrameAction
+ {
+ RAW, SKIP, UNCOMPRESS;
+ }
+
+ public static final class FrameMetaData
+ {
+ final int length;
+ final FrameAction frameAction;
+
+ /**
+ * @param frameAction
+ * @param length
+ */
+ public FrameMetaData(FrameAction frameAction, int length)
+ {
+ super();
+ this.frameAction = frameAction;
+ this.length = length;
+ }
+ }
+
+ public static final class FrameData
+ {
+ final int checkSum;
+ final int offset;
+
+ /**
+ * @param checkSum
+ * @param offset
+ */
+ public FrameData(int checkSum, int offset)
+ {
+ super();
+ this.checkSum = checkSum;
+ this.offset = offset;
+ }
+ }
+
+ private boolean ensureBuffer()
+ throws IOException
+ {
+ if (available() > 0) {
+ return true;
+ }
+ if (eof) {
+ return false;
+ }
+
+ if (!readBlockHeader()) {
+ eof = true;
+ return false;
+ }
+
+ // get action based on header
+ final FrameMetaData frameMetaData = getFrameMetaData(frameHeader);
+
+ if (FrameAction.SKIP == frameMetaData.frameAction) {
+ SnappyFramed.skip(rbc, frameMetaData.length,
+ ByteBuffer.wrap(buffer));
+ return ensureBuffer();
+ }
+
+ if (frameMetaData.length > input.capacity()) {
+ allocateBuffersBasedOnSize(frameMetaData.length);
+ }
+
+ input.clear();
+ input.limit(frameMetaData.length);
+
+ final int actualRead = readBytes(rbc, input);
+ if (actualRead != frameMetaData.length) {
+ throw new EOFException("unexpectd EOF when reading frame");
+ }
+ input.flip();
+
+ final FrameData frameData = getFrameData(input);
+
+ if (FrameAction.UNCOMPRESS == frameMetaData.frameAction) {
+
+ input.position(frameData.offset);
+
+ final int uncompressedLength = Snappy.uncompressedLength(input);
+
+ if (uncompressedLength > uncompressedDirect.capacity()) {
+ bufferPool.releaseDirect(uncompressedDirect);
+ bufferPool.releaseArray(buffer);
+ uncompressedDirect = bufferPool.allocateDirect(uncompressedLength);
+ buffer = bufferPool.allocateArray(uncompressedLength);
+ }
+
+ uncompressedDirect.clear();
+
+ this.valid = Snappy.uncompress(input, uncompressedDirect);
+
+ uncompressedDirect.get(buffer, 0, valid);
+ this.position = 0;
+ }
+ else {
+ // we need to start reading at the offset
+ input.position(frameData.offset);
+ this.position = 0;
+ this.valid = input.remaining();
+ this.input.get(buffer, 0, input.remaining());
+ }
+
+ if (verifyChecksums) {
+ final int actualCrc32c = SnappyFramed.maskedCrc32c(crc32, buffer,
+ position, valid - position);
+ if (frameData.checkSum != actualCrc32c) {
+ throw new IOException("Corrupt input: invalid checksum");
+ }
+ }
+
+ return true;
+ }
+
+ private boolean readBlockHeader()
+ throws IOException
+ {
+ frameHeader.clear();
+ int read = readBytes(rbc, frameHeader);
+
+ if (read == -1) {
+ return false;
+ }
+
+ if (read < frameHeader.capacity()) {
+ throw new EOFException("encountered EOF while reading block header");
+ }
+ frameHeader.flip();
+
+ return true;
+ }
+
+ /**
+ * @param frameHeader
+ * @return
+ * @throws IOException
+ */
+ private FrameMetaData getFrameMetaData(ByteBuffer frameHeader)
+ throws IOException
+ {
+
+ assert frameHeader.hasArray();
+
+ final byte[] frameHeaderArray = frameHeader.array();
+
+ int length = (frameHeaderArray[1] & 0xFF);
+ length |= (frameHeaderArray[2] & 0xFF) << 8;
+ length |= (frameHeaderArray[3] & 0xFF) << 16;
+
+ int minLength = 0;
+ final FrameAction frameAction;
+ final int flag = frameHeaderArray[0] & 0xFF;
+ switch (flag) {
+ case COMPRESSED_DATA_FLAG:
+ frameAction = FrameAction.UNCOMPRESS;
+ minLength = 5;
+ break;
+ case UNCOMPRESSED_DATA_FLAG:
+ frameAction = FrameAction.RAW;
+ minLength = 5;
+ break;
+ case STREAM_IDENTIFIER_FLAG:
+ if (length != 6) {
+ throw new IOException(
+ "stream identifier chunk with invalid length: "
+ + length);
+ }
+ frameAction = FrameAction.SKIP;
+ minLength = 6;
+ break;
+ default:
+ // Reserved unskippable chunks (chunk types 0x02-0x7f)
+ if (flag <= 0x7f) {
+ throw new IOException("unsupported unskippable chunk: "
+ + Integer.toHexString(flag));
+ }
+
+ // all that is left is Reserved skippable chunks (chunk types
+ // 0x80-0xfe)
+ frameAction = FrameAction.SKIP;
+ minLength = 0;
+ }
+
+ if (length < minLength) {
+ throw new IOException("invalid length: " + length
+ + " for chunk flag: " + Integer.toHexString(flag));
+ }
+
+ return new FrameMetaData(frameAction, length);
+ }
+
+ /**
+ * @param content
+ * @return
+ * @throws IOException
+ */
+ private FrameData getFrameData(ByteBuffer content)
+ throws IOException
+ {
+ return new FrameData(getCrc32c(content), 4);
+ }
+
+ private int getCrc32c(ByteBuffer content)
+ {
+
+ final int position = content.position();
+
+ return ((content.get(position + 3) & 0xFF) << 24)
+ | ((content.get(position + 2) & 0xFF) << 16)
+ | ((content.get(position + 1) & 0xFF) << 8)
+ | (content.get(position) & 0xFF);
+ }
+}
=====================================
src/main/java/org/xerial/snappy/SnappyFramedOutputStream.java
=====================================
@@ -1,568 +1,570 @@
-/*
- * Created: Apr 12, 2013
- */
-package org.xerial.snappy;
-
-import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
-import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
-import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
-import static org.xerial.snappy.SnappyFramed.maskedCrc32c;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.Channels;
-import java.nio.channels.ClosedChannelException;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-
-import org.xerial.snappy.pool.BufferPool;
-import org.xerial.snappy.pool.DefaultPoolFactory;
-
-/**
- * Implements the <a
- * href="http://snappy.googlecode.com/svn/trunk/framing_format.txt"
- * >x-snappy-framed</a> as an {@link OutputStream} and
- * {@link WritableByteChannel}.
- *
- * @author Brett Okken
- * @since 1.1.0
- */
-public final class SnappyFramedOutputStream
- extends OutputStream
- implements
- WritableByteChannel
-{
-
- /**
- * The x-snappy-framed specification allows for a chunk size up to
- * 16,777,211 bytes in length. However, it also goes on to state:
- * <p>
- * <code>
- * We place an additional restriction that the uncompressed data in a chunk
- * must be no longer than 65536 bytes. This allows consumers to easily use
- * small fixed-size buffers.
- * </code>
- * </p>
- */
- public static final int MAX_BLOCK_SIZE = 64 * 1024;
-
- /**
- * The default block size to use.
- */
- public static final int DEFAULT_BLOCK_SIZE = MAX_BLOCK_SIZE;
-
- /**
- * The default min compression ratio to use.
- */
- public static final double DEFAULT_MIN_COMPRESSION_RATIO = 0.85d;
-
- private final ByteBuffer headerBuffer = ByteBuffer.allocate(8).order(
- ByteOrder.LITTLE_ENDIAN);
- private final BufferPool bufferPool;
- private final int blockSize;
- private final ByteBuffer buffer;
- private final ByteBuffer directInputBuffer;
- private final ByteBuffer outputBuffer;
- private final double minCompressionRatio;
-
- private final WritableByteChannel out;
-
- // private int position;
- private boolean closed;
-
- /**
- * Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE}
- * and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
- * <p>
- * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
- * </p>
- *
- * @param out The underlying {@link OutputStream} to write to. Must not be
- * {@code null}.
- * @throws IOException
- */
- public SnappyFramedOutputStream(OutputStream out)
- throws IOException
- {
- this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, DefaultPoolFactory.getDefaultPool());
- }
-
- /**
- * Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE}
- * and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
- *
- * @param out The underlying {@link OutputStream} to write to. Must not be
- * {@code null}.
- * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedOutputStream(OutputStream out, BufferPool bufferPool)
- throws IOException
- {
- this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, bufferPool);
- }
-
- /**
- * Creates a new {@link SnappyFramedOutputStream} instance.
- * <p>
- * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
- * </p>
- *
- * @param out The underlying {@link OutputStream} to write to. Must not be
- * {@code null}.
- * @param blockSize The block size (of raw data) to compress before writing frames
- * to <i>out</i>. Must be in (0, 65536].
- * @param minCompressionRatio Defines the minimum compression ratio (
- * {@code compressedLength / rawLength}) that must be achieved to
- * write the compressed data. This must be in (0, 1.0].
- * @throws IOException
- */
- public SnappyFramedOutputStream(OutputStream out, int blockSize,
- double minCompressionRatio)
- throws IOException
- {
- this(Channels.newChannel(out), blockSize, minCompressionRatio, DefaultPoolFactory.getDefaultPool());
- }
-
- /**
- * Creates a new {@link SnappyFramedOutputStream} instance.
- *
- * @param out The underlying {@link OutputStream} to write to. Must not be
- * {@code null}.
- * @param blockSize The block size (of raw data) to compress before writing frames
- * to <i>out</i>. Must be in (0, 65536].
- * @param minCompressionRatio Defines the minimum compression ratio (
- * {@code compressedLength / rawLength}) that must be achieved to
- * write the compressed data. This must be in (0, 1.0].
- * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedOutputStream(OutputStream out, int blockSize,
- double minCompressionRatio, BufferPool bufferPool)
- throws IOException
- {
- this(Channels.newChannel(out), blockSize, minCompressionRatio, bufferPool);
- }
-
- /**
- * Creates a new {@link SnappyFramedOutputStream} using the
- * {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
- * <p>
- * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
- * </p>
- *
- * @param out The underlying {@link WritableByteChannel} to write to. Must
- * not be {@code null}.
- * @throws IOException
- * @since 1.1.1
- */
- public SnappyFramedOutputStream(WritableByteChannel out)
- throws IOException
- {
- this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, DefaultPoolFactory.getDefaultPool());
- }
-
- /**
- * Creates a new {@link SnappyFramedOutputStream} using the
- * {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
- * <p>
- * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
- * </p>
- *
- * @param out The underlying {@link WritableByteChannel} to write to. Must
- * not be {@code null}.
- * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
- * @throws IOException
- */
- public SnappyFramedOutputStream(WritableByteChannel out, BufferPool bufferPool)
- throws IOException
- {
- this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, bufferPool);
- }
-
- /**
- * Creates a new {@link SnappyFramedOutputStream} instance.
- *
- * @param out The underlying {@link WritableByteChannel} to write to. Must
- * not be {@code null}.
- * @param blockSize The block size (of raw data) to compress before writing frames
- * to <i>out</i>. Must be in (0, 65536].
- * @param minCompressionRatio Defines the minimum compression ratio (
- * {@code compressedLength / rawLength}) that must be achieved to
- * write the compressed data. This must be in (0, 1.0].
- * @throws IOException
- * @since 1.1.1
- */
- public SnappyFramedOutputStream(WritableByteChannel out, int blockSize,
- double minCompressionRatio)
- throws IOException
- {
- this(out, blockSize, minCompressionRatio, DefaultPoolFactory.getDefaultPool());
- }
-
- /**
- * Creates a new {@link SnappyFramedOutputStream} instance.
- *
- * @param out The underlying {@link WritableByteChannel} to write to. Must
- * not be {@code null}.
- * @param blockSize The block size (of raw data) to compress before writing frames
- * to <i>out</i>. Must be in (0, 65536].
- * @param minCompressionRatio Defines the minimum compression ratio (
- * {@code compressedLength / rawLength}) that must be achieved to
- * write the compressed data. This must be in (0, 1.0].
- * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
- * @throws IOException
- * @since 1.1.1
- */
- public SnappyFramedOutputStream(WritableByteChannel out, int blockSize,
- double minCompressionRatio, BufferPool bufferPool)
- throws IOException
- {
- if (out == null) {
- throw new NullPointerException("out is null");
- }
-
- if (bufferPool == null) {
- throw new NullPointerException("buffer pool is null");
- }
-
- if (minCompressionRatio <= 0 || minCompressionRatio > 1.0) {
- throw new IllegalArgumentException("minCompressionRatio "
- + minCompressionRatio + " must be in (0,1.0]");
- }
-
- if (blockSize <= 0 || blockSize > MAX_BLOCK_SIZE) {
- throw new IllegalArgumentException("block size " + blockSize
- + " must be in (0, 65536]");
- }
- this.blockSize = blockSize;
- this.out = out;
- this.minCompressionRatio = minCompressionRatio;
-
- this.bufferPool = bufferPool;
- buffer = ByteBuffer.wrap(bufferPool.allocateArray(blockSize), 0, blockSize);
- directInputBuffer = bufferPool.allocateDirect(blockSize);
- outputBuffer = bufferPool.allocateDirect(Snappy
- .maxCompressedLength(blockSize));
-
- writeHeader(out);
- }
-
- /**
- * Writes the implementation specific header or "marker bytes" to
- * <i>out</i>.
- *
- * @param out The underlying {@link OutputStream}.
- * @throws IOException
- */
- private void writeHeader(WritableByteChannel out)
- throws IOException
- {
- out.write(ByteBuffer.wrap(HEADER_BYTES));
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isOpen()
- {
- return !closed;
- }
-
- @Override
- public void write(int b)
- throws IOException
- {
- if (closed) {
- throw new IOException("Stream is closed");
- }
- if (buffer.remaining() <= 0) {
- flushBuffer();
- }
- buffer.put((byte) b);
- }
-
- @Override
- public void write(byte[] input, int offset, int length)
- throws IOException
- {
- if (closed) {
- throw new IOException("Stream is closed");
- }
-
- if (input == null) {
- throw new NullPointerException();
- }
- else if ((offset < 0) || (offset > input.length) || (length < 0)
- || ((offset + length) > input.length)
- || ((offset + length) < 0)) {
- throw new IndexOutOfBoundsException();
- }
-
- while (length > 0) {
- if (buffer.remaining() <= 0) {
- flushBuffer();
- }
-
- final int toPut = Math.min(length, buffer.remaining());
- buffer.put(input, offset, toPut);
- offset += toPut;
- length -= toPut;
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public int write(ByteBuffer src)
- throws IOException
- {
- if (closed) {
- throw new ClosedChannelException();
- }
-
- if (buffer.remaining() <= 0) {
- flushBuffer();
- }
-
- final int srcLength = src.remaining();
-
- // easy case: enough free space in buffer for entire input
- if (buffer.remaining() >= src.remaining()) {
- buffer.put(src);
- return srcLength;
- }
-
- // store current limit
- final int srcEnd = src.position() + src.remaining();
-
- while ((src.position() + buffer.remaining()) <= srcEnd) {
- // fill partial buffer as much as possible and flush
- src.limit(src.position() + buffer.remaining());
- buffer.put(src);
- flushBuffer();
- }
-
- // reset original limit
- src.limit(srcEnd);
-
- // copy remaining partial block into now-empty buffer
- buffer.put(src);
-
- return srcLength;
- }
-
- /**
- * Transfers all the content from <i>is</i> to this {@link OutputStream}.
- * This potentially limits the amount of buffering required to compress
- * content.
- *
- * @param is The source of data to compress.
- * @return The number of bytes read from <i>is</i>.
- * @throws IOException
- * @since 1.1.1
- */
- public long transferFrom(InputStream is)
- throws IOException
- {
- if (closed) {
- throw new ClosedChannelException();
- }
-
- if (is == null) {
- throw new NullPointerException();
- }
-
- if (buffer.remaining() == 0) {
- flushBuffer();
- }
-
- assert buffer.hasArray();
- final byte[] bytes = buffer.array();
-
- final int arrayOffset = buffer.arrayOffset();
- long totTransfered = 0;
- int read;
- while ((read = is.read(bytes, arrayOffset + buffer.position(),
- buffer.remaining())) != -1) {
- buffer.position(buffer.position() + read);
-
- if (buffer.remaining() == 0) {
- flushBuffer();
- }
-
- totTransfered += read;
- }
-
- return totTransfered;
- }
-
- /**
- * Transfers all the content from <i>rbc</i> to this
- * {@link WritableByteChannel}. This potentially limits the amount of
- * buffering required to compress content.
- *
- * @param rbc The source of data to compress.
- * @return The number of bytes read from <i>rbc</i>.
- * @throws IOException
- * @since 1.1.1
- */
- public long transferFrom(ReadableByteChannel rbc)
- throws IOException
- {
- if (closed) {
- throw new ClosedChannelException();
- }
-
- if (rbc == null) {
- throw new NullPointerException();
- }
-
- if (buffer.remaining() == 0) {
- flushBuffer();
- }
-
- long totTransfered = 0;
- int read;
- while ((read = rbc.read(buffer)) != -1) {
- if (buffer.remaining() == 0) {
- flushBuffer();
- }
-
- totTransfered += read;
- }
-
- return totTransfered;
- }
-
- @Override
- public final void flush()
- throws IOException
- {
- if (closed) {
- throw new IOException("Stream is closed");
- }
- flushBuffer();
- }
-
- @Override
- public final void close()
- throws IOException
- {
- if (closed) {
- return;
- }
- try {
- flush();
- out.close();
- }
- finally {
- closed = true;
- bufferPool.releaseArray(buffer.array());
- bufferPool.releaseDirect(directInputBuffer);
- bufferPool.releaseDirect(outputBuffer);
- }
- }
-
- /**
- * Compresses and writes out any buffered data. This does nothing if there
- * is no currently buffered data.
- *
- * @throws IOException
- */
- private void flushBuffer()
- throws IOException
- {
- if (buffer.position() > 0) {
- buffer.flip();
- writeCompressed(buffer);
- buffer.clear();
- buffer.limit(blockSize);
- }
- }
-
- /**
- * {@link SnappyFramed#maskedCrc32c(byte[], int, int)} the crc, compresses
- * the data, determines if the compression ratio is acceptable and calls
- * {@link #writeBlock(java.nio.channels.WritableByteChannel, java.nio.ByteBuffer, boolean, int)} to
- * actually write the frame.
- *
- * @param buffer
- * @throws IOException
- */
- private void writeCompressed(ByteBuffer buffer)
- throws IOException
- {
-
- final byte[] input = buffer.array();
- final int length = buffer.remaining();
-
- // crc is based on the user supplied input data
- final int crc32c = maskedCrc32c(input, 0, length);
-
- directInputBuffer.clear();
- directInputBuffer.put(buffer);
- directInputBuffer.flip();
-
- outputBuffer.clear();
- Snappy.compress(directInputBuffer, outputBuffer);
-
- final int compressedLength = outputBuffer.remaining();
-
- // only use the compressed data if compression ratio is <= the
- // minCompressonRatio
- if (((double) compressedLength / (double) length) <= minCompressionRatio) {
- writeBlock(out, outputBuffer, true, crc32c);
- }
- else {
- // otherwise use the uncompressed data.
- buffer.flip();
- writeBlock(out, buffer, false, crc32c);
- }
- }
-
- /**
- * Write a frame (block) to <i>out</i>.
- *
- * @param out The {@link OutputStream} to write to.
- * @param data The data to write.
- * @param compressed Indicates if <i>data</i> is the compressed or raw content.
- * This is based on whether the compression ratio desired is
- * reached.
- * @param crc32c The calculated checksum.
- * @throws IOException
- */
- private void writeBlock(final WritableByteChannel out, ByteBuffer data,
- boolean compressed, int crc32c)
- throws IOException
- {
-
- headerBuffer.clear();
- headerBuffer.put((byte) (compressed ? COMPRESSED_DATA_FLAG
- : UNCOMPRESSED_DATA_FLAG));
-
- // the length written out to the header is both the checksum and the
- // frame
- final int headerLength = data.remaining() + 4;
-
- // write length
- headerBuffer.put((byte) headerLength);
- headerBuffer.put((byte) (headerLength >>> 8));
- headerBuffer.put((byte) (headerLength >>> 16));
-
- // write crc32c of user input data
- headerBuffer.putInt(crc32c);
-
- headerBuffer.flip();
-
- // write the header
- out.write(headerBuffer);
- // write the raw data
- out.write(data);
- }
-}
+/*
+ * Created: Apr 12, 2013
+ */
+package org.xerial.snappy;
+
+import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
+import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
+import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
+import static org.xerial.snappy.SnappyFramed.maskedCrc32c;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.Channels;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.util.zip.Checksum;
+
+import org.xerial.snappy.pool.BufferPool;
+import org.xerial.snappy.pool.DefaultPoolFactory;
+
+/**
+ * Implements the <a
+ * href="http://snappy.googlecode.com/svn/trunk/framing_format.txt"
+ * >x-snappy-framed</a> as an {@link OutputStream} and
+ * {@link WritableByteChannel}.
+ *
+ * @author Brett Okken
+ * @since 1.1.0
+ */
+public final class SnappyFramedOutputStream
+ extends OutputStream
+ implements
+ WritableByteChannel
+{
+
+ /**
+ * The x-snappy-framed specification allows for a chunk size up to
+ * 16,777,211 bytes in length. However, it also goes on to state:
+ * <p>
+ * <code>
+ * We place an additional restriction that the uncompressed data in a chunk
+ * must be no longer than 65536 bytes. This allows consumers to easily use
+ * small fixed-size buffers.
+ * </code>
+ * </p>
+ */
+ public static final int MAX_BLOCK_SIZE = 64 * 1024;
+
+ /**
+ * The default block size to use.
+ */
+ public static final int DEFAULT_BLOCK_SIZE = MAX_BLOCK_SIZE;
+
+ /**
+ * The default min compression ratio to use.
+ */
+ public static final double DEFAULT_MIN_COMPRESSION_RATIO = 0.85d;
+
+ private final Checksum crc32 = SnappyFramed.getCRC32C();
+ private final ByteBuffer headerBuffer = ByteBuffer.allocate(8).order(
+ ByteOrder.LITTLE_ENDIAN);
+ private final BufferPool bufferPool;
+ private final int blockSize;
+ private final ByteBuffer buffer;
+ private final ByteBuffer directInputBuffer;
+ private final ByteBuffer outputBuffer;
+ private final double minCompressionRatio;
+
+ private final WritableByteChannel out;
+
+ // private int position;
+ private boolean closed;
+
+ /**
+ * Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE}
+ * and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
+ * <p>
+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
+ * </p>
+ *
+ * @param out The underlying {@link OutputStream} to write to. Must not be
+ * {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedOutputStream(OutputStream out)
+ throws IOException
+ {
+ this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, DefaultPoolFactory.getDefaultPool());
+ }
+
+ /**
+ * Creates a new {@link SnappyFramedOutputStream} using the {@link #DEFAULT_BLOCK_SIZE}
+ * and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
+ *
+ * @param out The underlying {@link OutputStream} to write to. Must not be
+ * {@code null}.
+ * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedOutputStream(OutputStream out, BufferPool bufferPool)
+ throws IOException
+ {
+ this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, bufferPool);
+ }
+
+ /**
+ * Creates a new {@link SnappyFramedOutputStream} instance.
+ * <p>
+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
+ * </p>
+ *
+ * @param out The underlying {@link OutputStream} to write to. Must not be
+ * {@code null}.
+ * @param blockSize The block size (of raw data) to compress before writing frames
+ * to <i>out</i>. Must be in (0, 65536].
+ * @param minCompressionRatio Defines the minimum compression ratio (
+ * {@code compressedLength / rawLength}) that must be achieved to
+ * write the compressed data. This must be in (0, 1.0].
+ * @throws IOException
+ */
+ public SnappyFramedOutputStream(OutputStream out, int blockSize,
+ double minCompressionRatio)
+ throws IOException
+ {
+ this(Channels.newChannel(out), blockSize, minCompressionRatio, DefaultPoolFactory.getDefaultPool());
+ }
+
+ /**
+ * Creates a new {@link SnappyFramedOutputStream} instance.
+ *
+ * @param out The underlying {@link OutputStream} to write to. Must not be
+ * {@code null}.
+ * @param blockSize The block size (of raw data) to compress before writing frames
+ * to <i>out</i>. Must be in (0, 65536].
+ * @param minCompressionRatio Defines the minimum compression ratio (
+ * {@code compressedLength / rawLength}) that must be achieved to
+ * write the compressed data. This must be in (0, 1.0].
+ * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedOutputStream(OutputStream out, int blockSize,
+ double minCompressionRatio, BufferPool bufferPool)
+ throws IOException
+ {
+ this(Channels.newChannel(out), blockSize, minCompressionRatio, bufferPool);
+ }
+
+ /**
+ * Creates a new {@link SnappyFramedOutputStream} using the
+ * {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
+ * <p>
+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
+ * </p>
+ *
+ * @param out The underlying {@link WritableByteChannel} to write to. Must
+ * not be {@code null}.
+ * @throws IOException
+ * @since 1.1.1
+ */
+ public SnappyFramedOutputStream(WritableByteChannel out)
+ throws IOException
+ {
+ this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, DefaultPoolFactory.getDefaultPool());
+ }
+
+ /**
+ * Creates a new {@link SnappyFramedOutputStream} using the
+ * {@link #DEFAULT_BLOCK_SIZE} and {@link #DEFAULT_MIN_COMPRESSION_RATIO}.
+ * <p>
+ * Uses {@link DefaultPoolFactory} to obtain {@link BufferPool} for buffers.
+ * </p>
+ *
+ * @param out The underlying {@link WritableByteChannel} to write to. Must
+ * not be {@code null}.
+ * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
+ * @throws IOException
+ */
+ public SnappyFramedOutputStream(WritableByteChannel out, BufferPool bufferPool)
+ throws IOException
+ {
+ this(out, DEFAULT_BLOCK_SIZE, DEFAULT_MIN_COMPRESSION_RATIO, bufferPool);
+ }
+
+ /**
+ * Creates a new {@link SnappyFramedOutputStream} instance.
+ *
+ * @param out The underlying {@link WritableByteChannel} to write to. Must
+ * not be {@code null}.
+ * @param blockSize The block size (of raw data) to compress before writing frames
+ * to <i>out</i>. Must be in (0, 65536].
+ * @param minCompressionRatio Defines the minimum compression ratio (
+ * {@code compressedLength / rawLength}) that must be achieved to
+ * write the compressed data. This must be in (0, 1.0].
+ * @throws IOException
+ * @since 1.1.1
+ */
+ public SnappyFramedOutputStream(WritableByteChannel out, int blockSize,
+ double minCompressionRatio)
+ throws IOException
+ {
+ this(out, blockSize, minCompressionRatio, DefaultPoolFactory.getDefaultPool());
+ }
+
+ /**
+ * Creates a new {@link SnappyFramedOutputStream} instance.
+ *
+ * @param out The underlying {@link WritableByteChannel} to write to. Must
+ * not be {@code null}.
+ * @param blockSize The block size (of raw data) to compress before writing frames
+ * to <i>out</i>. Must be in (0, 65536].
+ * @param minCompressionRatio Defines the minimum compression ratio (
+ * {@code compressedLength / rawLength}) that must be achieved to
+ * write the compressed data. This must be in (0, 1.0].
+ * @param bufferPool Used to obtain buffer instances. Must not be {@code null}.
+ * @throws IOException
+ * @since 1.1.1
+ */
+ public SnappyFramedOutputStream(WritableByteChannel out, int blockSize,
+ double minCompressionRatio, BufferPool bufferPool)
+ throws IOException
+ {
+ if (out == null) {
+ throw new NullPointerException("out is null");
+ }
+
+ if (bufferPool == null) {
+ throw new NullPointerException("buffer pool is null");
+ }
+
+ if (minCompressionRatio <= 0 || minCompressionRatio > 1.0) {
+ throw new IllegalArgumentException("minCompressionRatio "
+ + minCompressionRatio + " must be in (0,1.0]");
+ }
+
+ if (blockSize <= 0 || blockSize > MAX_BLOCK_SIZE) {
+ throw new IllegalArgumentException("block size " + blockSize
+ + " must be in (0, 65536]");
+ }
+ this.blockSize = blockSize;
+ this.out = out;
+ this.minCompressionRatio = minCompressionRatio;
+
+ this.bufferPool = bufferPool;
+ buffer = ByteBuffer.wrap(bufferPool.allocateArray(blockSize), 0, blockSize);
+ directInputBuffer = bufferPool.allocateDirect(blockSize);
+ outputBuffer = bufferPool.allocateDirect(Snappy
+ .maxCompressedLength(blockSize));
+
+ writeHeader(out);
+ }
+
+ /**
+ * Writes the implementation specific header or "marker bytes" to
+ * <i>out</i>.
+ *
+ * @param out The underlying {@link OutputStream}.
+ * @throws IOException
+ */
+ private void writeHeader(WritableByteChannel out)
+ throws IOException
+ {
+ out.write(ByteBuffer.wrap(HEADER_BYTES));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isOpen()
+ {
+ return !closed;
+ }
+
+ @Override
+ public void write(int b)
+ throws IOException
+ {
+ if (closed) {
+ throw new IOException("Stream is closed");
+ }
+ if (buffer.remaining() <= 0) {
+ flushBuffer();
+ }
+ buffer.put((byte) b);
+ }
+
+ @Override
+ public void write(byte[] input, int offset, int length)
+ throws IOException
+ {
+ if (closed) {
+ throw new IOException("Stream is closed");
+ }
+
+ if (input == null) {
+ throw new NullPointerException();
+ }
+ else if ((offset < 0) || (offset > input.length) || (length < 0)
+ || ((offset + length) > input.length)
+ || ((offset + length) < 0)) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ while (length > 0) {
+ if (buffer.remaining() <= 0) {
+ flushBuffer();
+ }
+
+ final int toPut = Math.min(length, buffer.remaining());
+ buffer.put(input, offset, toPut);
+ offset += toPut;
+ length -= toPut;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int write(ByteBuffer src)
+ throws IOException
+ {
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ if (buffer.remaining() <= 0) {
+ flushBuffer();
+ }
+
+ final int srcLength = src.remaining();
+
+ // easy case: enough free space in buffer for entire input
+ if (buffer.remaining() >= src.remaining()) {
+ buffer.put(src);
+ return srcLength;
+ }
+
+ // store current limit
+ final int srcEnd = src.position() + src.remaining();
+
+ while ((src.position() + buffer.remaining()) <= srcEnd) {
+ // fill partial buffer as much as possible and flush
+ src.limit(src.position() + buffer.remaining());
+ buffer.put(src);
+ flushBuffer();
+ }
+
+ // reset original limit
+ src.limit(srcEnd);
+
+ // copy remaining partial block into now-empty buffer
+ buffer.put(src);
+
+ return srcLength;
+ }
+
+ /**
+ * Transfers all the content from <i>is</i> to this {@link OutputStream}.
+ * This potentially limits the amount of buffering required to compress
+ * content.
+ *
+ * @param is The source of data to compress.
+ * @return The number of bytes read from <i>is</i>.
+ * @throws IOException
+ * @since 1.1.1
+ */
+ public long transferFrom(InputStream is)
+ throws IOException
+ {
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ if (is == null) {
+ throw new NullPointerException();
+ }
+
+ if (buffer.remaining() == 0) {
+ flushBuffer();
+ }
+
+ assert buffer.hasArray();
+ final byte[] bytes = buffer.array();
+
+ final int arrayOffset = buffer.arrayOffset();
+ long totTransfered = 0;
+ int read;
+ while ((read = is.read(bytes, arrayOffset + buffer.position(),
+ buffer.remaining())) != -1) {
+ buffer.position(buffer.position() + read);
+
+ if (buffer.remaining() == 0) {
+ flushBuffer();
+ }
+
+ totTransfered += read;
+ }
+
+ return totTransfered;
+ }
+
+ /**
+ * Transfers all the content from <i>rbc</i> to this
+ * {@link WritableByteChannel}. This potentially limits the amount of
+ * buffering required to compress content.
+ *
+ * @param rbc The source of data to compress.
+ * @return The number of bytes read from <i>rbc</i>.
+ * @throws IOException
+ * @since 1.1.1
+ */
+ public long transferFrom(ReadableByteChannel rbc)
+ throws IOException
+ {
+ if (closed) {
+ throw new ClosedChannelException();
+ }
+
+ if (rbc == null) {
+ throw new NullPointerException();
+ }
+
+ if (buffer.remaining() == 0) {
+ flushBuffer();
+ }
+
+ long totTransfered = 0;
+ int read;
+ while ((read = rbc.read(buffer)) != -1) {
+ if (buffer.remaining() == 0) {
+ flushBuffer();
+ }
+
+ totTransfered += read;
+ }
+
+ return totTransfered;
+ }
+
+ @Override
+ public final void flush()
+ throws IOException
+ {
+ if (closed) {
+ throw new IOException("Stream is closed");
+ }
+ flushBuffer();
+ }
+
+ @Override
+ public final void close()
+ throws IOException
+ {
+ if (closed) {
+ return;
+ }
+ try {
+ flush();
+ out.close();
+ }
+ finally {
+ closed = true;
+ bufferPool.releaseArray(buffer.array());
+ bufferPool.releaseDirect(directInputBuffer);
+ bufferPool.releaseDirect(outputBuffer);
+ }
+ }
+
+ /**
+ * Compresses and writes out any buffered data. This does nothing if there
+ * is no currently buffered data.
+ *
+ * @throws IOException
+ */
+ private void flushBuffer()
+ throws IOException
+ {
+ if (buffer.position() > 0) {
+ buffer.flip();
+ writeCompressed(buffer);
+ buffer.clear();
+ buffer.limit(blockSize);
+ }
+ }
+
+ /**
+ * {@link SnappyFramed#maskedCrc32c(byte[], int, int)} the crc, compresses
+ * the data, determines if the compression ratio is acceptable and calls
+ * {@link #writeBlock(java.nio.channels.WritableByteChannel, java.nio.ByteBuffer, boolean, int)} to
+ * actually write the frame.
+ *
+ * @param buffer
+ * @throws IOException
+ */
+ private void writeCompressed(ByteBuffer buffer)
+ throws IOException
+ {
+
+ final byte[] input = buffer.array();
+ final int length = buffer.remaining();
+
+ // crc is based on the user supplied input data
+ final int crc32c = maskedCrc32c(crc32, input, 0, length);
+
+ directInputBuffer.clear();
+ directInputBuffer.put(buffer);
+ directInputBuffer.flip();
+
+ outputBuffer.clear();
+ Snappy.compress(directInputBuffer, outputBuffer);
+
+ final int compressedLength = outputBuffer.remaining();
+
+ // only use the compressed data if compression ratio is <= the
+ // minCompressonRatio
+ if (((double) compressedLength / (double) length) <= minCompressionRatio) {
+ writeBlock(out, outputBuffer, true, crc32c);
+ }
+ else {
+ // otherwise use the uncompressed data.
+ buffer.flip();
+ writeBlock(out, buffer, false, crc32c);
+ }
+ }
+
+ /**
+ * Write a frame (block) to <i>out</i>.
+ *
+ * @param out The {@link OutputStream} to write to.
+ * @param data The data to write.
+ * @param compressed Indicates if <i>data</i> is the compressed or raw content.
+ * This is based on whether the compression ratio desired is
+ * reached.
+ * @param crc32c The calculated checksum.
+ * @throws IOException
+ */
+ private void writeBlock(final WritableByteChannel out, ByteBuffer data,
+ boolean compressed, int crc32c)
+ throws IOException
+ {
+
+ headerBuffer.clear();
+ headerBuffer.put((byte) (compressed ? COMPRESSED_DATA_FLAG
+ : UNCOMPRESSED_DATA_FLAG));
+
+ // the length written out to the header is both the checksum and the
+ // frame
+ final int headerLength = data.remaining() + 4;
+
+ // write length
+ headerBuffer.put((byte) headerLength);
+ headerBuffer.put((byte) (headerLength >>> 8));
+ headerBuffer.put((byte) (headerLength >>> 16));
+
+ // write crc32c of user input data
+ headerBuffer.putInt(crc32c);
+
+ headerBuffer.flip();
+
+ // write the header
+ out.write(headerBuffer);
+ // write the raw data
+ out.write(data);
+ }
+}
=====================================
src/main/java/org/xerial/snappy/SnappyLoader.java
=====================================
@@ -168,7 +168,7 @@ public class SnappyLoader
setSnappyApi(new SnappyNative());
}
}
- catch(Exception e) {
+ catch(Throwable e) {
// Fall-back to pure-java Snappy implementation
setSnappyApi(new PureJavaSnappy());
}
@@ -348,7 +348,7 @@ public class SnappyLoader
if (!hasNativeLib) {
if (OSInfo.getOSName().equals("Mac")) {
// Fix for openjdk7 for Mac
- String altName = "libsnappyjava.jnilib";
+ String altName = "libsnappyjava.dylib";
if (hasResource(snappyNativeLibraryPath + "/" + altName)) {
snappyNativeLibraryName = altName;
hasNativeLib = true;
=====================================
src/main/java/org/xerial/snappy/pure/PureJavaSnappy.java
=====================================
@@ -1,26 +1,41 @@
package org.xerial.snappy.pure;
-import org.xerial.snappy.SnappyApi;
+import static org.xerial.snappy.pure.UnsafeUtil.getAddress;
+import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET;
import java.io.IOException;
+import java.lang.ref.SoftReference;
import java.nio.ByteBuffer;
+import java.util.concurrent.ConcurrentLinkedDeque;
-import static org.xerial.snappy.pure.UnsafeUtil.getAddress;
-import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET;
+import org.xerial.snappy.SnappyApi;
/**
* A pure-java Snappy implementation using https://github.com/airlift/aircompressor
*/
public class PureJavaSnappy implements SnappyApi
{
- private final short[] table = new short[SnappyRawCompressor.MAX_HASH_TABLE_SIZE];
+ /**
+ * Using a {@link ConcurrentLinkedDeque}, with values constantly popped and pushed from the head, leads to the fewest
+ * {@code short[]} instances remaining live over time.
+ */
+ private final static ConcurrentLinkedDeque<SoftReference<short[]>> CACHED_TABLES = new ConcurrentLinkedDeque<>();
+
private final static int MAX_OUTPUT_LENGTH = Integer.MAX_VALUE;
@Override
public long rawCompress(long inputAddr, long inputSize, long destAddr)
throws IOException
{
- return SnappyRawCompressor.compress(null, inputAddr, inputSize, null, destAddr, MAX_OUTPUT_LENGTH, table);
+ final short[] table = getTable();
+ try
+ {
+ return SnappyRawCompressor.compress(null, inputAddr, inputSize, null, destAddr, MAX_OUTPUT_LENGTH, table);
+ }
+ finally
+ {
+ returnTable(table);
+ }
}
@Override
@@ -76,16 +91,24 @@ public class PureJavaSnappy implements SnappyApi
// collected in a block, and technically, the JVM is allowed to eliminate these locks.
synchronized (input) {
synchronized (compressed) {
- int written = SnappyRawCompressor.compress(
- inputBase,
- inputAddress,
- inputLimit,
- outputBase,
- outputAddress,
- outputLimit,
- table);
- compressed.position(compressed.position() + written);
- return written;
+ final short[] table = getTable();
+ try
+ {
+ int written = SnappyRawCompressor.compress(
+ inputBase,
+ inputAddress,
+ inputLimit,
+ outputBase,
+ outputAddress,
+ outputLimit,
+ table);
+ compressed.position(compressed.position() + written);
+ return written;
+ }
+ finally
+ {
+ returnTable(table);
+ }
}
}
}
@@ -99,7 +122,15 @@ public class PureJavaSnappy implements SnappyApi
long outputAddress = ARRAY_BYTE_BASE_OFFSET + outputOffset;
long outputLimit = outputAddress + MAX_OUTPUT_LENGTH;
- return SnappyRawCompressor.compress(input, inputAddress, inputLimit, output, outputAddress, outputLimit, table);
+ final short[] table = getTable();
+ try
+ {
+ return SnappyRawCompressor.compress(input, inputAddress, inputLimit, output, outputAddress, outputLimit, table);
+ }
+ finally
+ {
+ returnTable(table);
+ }
}
@Override
@@ -241,4 +272,38 @@ public class PureJavaSnappy implements SnappyApi
{
System.arraycopy(src, offset, dest, dOffset, byteLength);
}
+
+ private static short[] getTable()
+ {
+ SoftReference<short[]> existingRef;
+ while((existingRef = CACHED_TABLES.poll()) != null)
+ {
+ short[] table = existingRef.get();
+ if (table != null)
+ {
+ //purge oldest entries have lost references
+ SoftReference<short[]> entry;
+ boolean lastEmpty = true;
+ while (lastEmpty && (entry = CACHED_TABLES.peekLast()) != null)
+ {
+ if (entry.get() == null)
+ {
+ CACHED_TABLES.removeLastOccurrence(entry);
+ }
+ else
+ {
+ lastEmpty = false;
+ }
+ }
+
+ return table;
+ }
+ }
+ return new short[SnappyRawCompressor.MAX_HASH_TABLE_SIZE];
+ }
+
+ private static void returnTable(short[] table)
+ {
+ CACHED_TABLES.addFirst(new SoftReference<short[]>(table));
+ }
}
=====================================
src/main/java/org/xerial/snappy/pure/SnappyRawCompressor.java
=====================================
@@ -12,6 +12,8 @@
* limitations under the License.
*/
package org.xerial.snappy.pure;
+
+import java.nio.ByteOrder;
import java.util.Arrays;
import static org.xerial.snappy.pure.SnappyConstants.COPY_1_BYTE_OFFSET;
@@ -20,6 +22,9 @@ import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_INT;
import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_LONG;
import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_SHORT;
import static org.xerial.snappy.pure.UnsafeUtil.UNSAFE;
+import static java.lang.Integer.reverseBytes;
+import static java.lang.Long.reverseBytes;
+import static java.lang.Short.reverseBytes;
public final class SnappyRawCompressor
{
@@ -40,8 +45,25 @@ public final class SnappyRawCompressor
private static final int MAX_HASH_TABLE_BITS = 14;
public static final int MAX_HASH_TABLE_SIZE = 1 << MAX_HASH_TABLE_BITS;
+ private static final ByteOrder byteOrder = ByteOrder.nativeOrder();
+
private SnappyRawCompressor() {}
+ private static int littleEndian(int i)
+ {
+ return (byteOrder == ByteOrder.LITTLE_ENDIAN) ? i : reverseBytes(i);
+ }
+
+ private static long littleEndian(long i)
+ {
+ return (byteOrder == ByteOrder.LITTLE_ENDIAN) ? i : reverseBytes(i);
+ }
+
+ private static short littleEndian(short i)
+ {
+ return (byteOrder == ByteOrder.LITTLE_ENDIAN) ? i : reverseBytes(i);
+ }
+
public static int maxCompressedLength(int sourceLength)
{
// Compressed data can be defined as:
@@ -138,7 +160,7 @@ public final class SnappyRawCompressor
long candidateIndex = 0;
for (input += 1; input + (skip >>> 5) <= fastInputLimit; input += ((skip++) >>> 5)) {
// hash the 4 bytes starting at the input pointer
- int currentInt = UNSAFE.getInt(inputBase, input);
+ int currentInt = littleEndian(UNSAFE.getInt(inputBase, input));
int hash = hashBytes(currentInt, shift);
// get the position of a 4 bytes sequence with the same hash
@@ -151,7 +173,7 @@ public final class SnappyRawCompressor
// if the 4 byte sequence a the candidate index matches the sequence at the
// current position, proceed to the next phase
- if (currentInt == UNSAFE.getInt(inputBase, candidateIndex)) {
+ if (currentInt == littleEndian(UNSAFE.getInt(inputBase, candidateIndex))) {
break;
}
}
@@ -201,7 +223,7 @@ public final class SnappyRawCompressor
// We could immediately start working at input now, but to improve
// compression we first update table[Hash(ip - 1, ...)].
- long longValue = UNSAFE.getLong(inputBase, input - 1);
+ long longValue = littleEndian((UNSAFE.getLong(inputBase, input - 1)));
int prevInt = (int) longValue;
inputBytes = (int) (longValue >>> 8);
@@ -214,7 +236,7 @@ public final class SnappyRawCompressor
candidateIndex = blockAddress + (table[curHash] & 0xFFFF);
table[curHash] = (short) (input - blockAddress);
- } while (inputBytes == UNSAFE.getInt(inputBase, candidateIndex));
+ } while (inputBytes == littleEndian(UNSAFE.getInt(inputBase, candidateIndex)));
nextEmitAddress = input;
}
@@ -236,7 +258,7 @@ public final class SnappyRawCompressor
// first, compare long at a time
while (current < matchLimit - (SIZE_OF_LONG - 1)) {
- long diff = UNSAFE.getLong(inputBase, matchStart) ^ UNSAFE.getLong(inputBase, current);
+ long diff = littleEndian(UNSAFE.getLong(inputBase, matchStart)) ^ littleEndian(UNSAFE.getLong(inputBase, current));
if (diff != 0) {
current += Long.numberOfTrailingZeros(diff) >> 3;
return (int) (current - start);
@@ -246,12 +268,12 @@ public final class SnappyRawCompressor
matchStart += SIZE_OF_LONG;
}
- if (current < matchLimit - (SIZE_OF_INT - 1) && UNSAFE.getInt(inputBase, matchStart) == UNSAFE.getInt(inputBase, current)) {
+ if (current < matchLimit - (SIZE_OF_INT - 1) && littleEndian(UNSAFE.getInt(inputBase, matchStart)) == littleEndian(UNSAFE.getInt(inputBase, current))) {
current += SIZE_OF_INT;
matchStart += SIZE_OF_INT;
}
- if (current < matchLimit - (SIZE_OF_SHORT - 1) && UNSAFE.getShort(inputBase, matchStart) == UNSAFE.getShort(inputBase, current)) {
+ if (current < matchLimit - (SIZE_OF_SHORT - 1) && littleEndian(UNSAFE.getShort(inputBase, matchStart)) == littleEndian(UNSAFE.getShort(inputBase, current))) {
current += SIZE_OF_SHORT;
matchStart += SIZE_OF_SHORT;
}
@@ -289,7 +311,7 @@ public final class SnappyRawCompressor
bytes = 4;
}
// System is assumed to be little endian, so low bytes will be zero for the smaller numbers
- UNSAFE.putInt(outputBase, output, n);
+ UNSAFE.putInt(outputBase, output, littleEndian(n));
output += bytes;
}
return output;
@@ -314,7 +336,7 @@ public final class SnappyRawCompressor
// Emit 64 byte copies but make sure to keep at least four bytes reserved
while (matchLength >= 68) {
UNSAFE.putByte(outputBase, output++, (byte) (COPY_2_BYTE_OFFSET + ((64 - 1) << 2)));
- UNSAFE.putShort(outputBase, output, (short) offset);
+ UNSAFE.putShort(outputBase, output, littleEndian((short) offset));
output += SIZE_OF_SHORT;
matchLength -= 64;
}
@@ -323,7 +345,7 @@ public final class SnappyRawCompressor
// length < 68
if (matchLength > 64) {
UNSAFE.putByte(outputBase, output++, (byte) (COPY_2_BYTE_OFFSET + ((60 - 1) << 2)));
- UNSAFE.putShort(outputBase, output, (short) offset);
+ UNSAFE.putShort(outputBase, output, littleEndian((short) offset));
output += SIZE_OF_SHORT;
matchLength -= 60;
}
@@ -336,7 +358,7 @@ public final class SnappyRawCompressor
}
else {
UNSAFE.putByte(outputBase, output++, (byte) (COPY_2_BYTE_OFFSET + ((matchLength - 1) << 2)));
- UNSAFE.putShort(outputBase, output, (short) offset);
+ UNSAFE.putShort(outputBase, output, littleEndian((short) offset));
output += SIZE_OF_SHORT;
}
return output;
=====================================
src/main/java/org/xerial/snappy/pure/SnappyRawDecompressor.java
=====================================
@@ -13,6 +13,8 @@
*/
package org.xerial.snappy.pure;
+import java.nio.ByteOrder;
+
import org.xerial.snappy.SnappyError;
import org.xerial.snappy.SnappyErrorCode;
@@ -20,6 +22,7 @@ import static org.xerial.snappy.pure.SnappyConstants.LITERAL;
import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_INT;
import static org.xerial.snappy.pure.SnappyConstants.SIZE_OF_LONG;
import static org.xerial.snappy.pure.UnsafeUtil.UNSAFE;
+import static java.lang.Integer.reverseBytes;
public final class SnappyRawDecompressor
{
@@ -28,6 +31,12 @@ public final class SnappyRawDecompressor
private SnappyRawDecompressor() {}
+ private static final ByteOrder byteOrder = ByteOrder.nativeOrder();
+
+ private static int littleEndian(int i) {
+ return (byteOrder == ByteOrder.LITTLE_ENDIAN) ? i : reverseBytes(i);
+ }
+
public static int getUncompressedLength(Object compressed, long compressedAddress, long compressedLimit)
{
return readUncompressedLength(compressed, compressedAddress, compressedLimit)[0];
@@ -89,7 +98,7 @@ public final class SnappyRawDecompressor
int trailerBytes = entry >>> 11;
int trailer = 0;
if (input + SIZE_OF_INT < inputLimit) {
- trailer = UNSAFE.getInt(inputBase, input) & wordmask[trailerBytes];
+ trailer = littleEndian(UNSAFE.getInt(inputBase, input)) & wordmask[trailerBytes];
}
else {
if (input + trailerBytes > inputLimit) {
=====================================
src/main/java/org/xerial/snappy/pure/UnsafeUtil.java
=====================================
@@ -19,24 +19,17 @@ import sun.misc.Unsafe;
import java.lang.reflect.Field;
import java.nio.Buffer;
-import java.nio.ByteOrder;
-
-import static java.lang.String.format;
final class UnsafeUtil
{
public static final Unsafe UNSAFE;
private static final Field ADDRESS_ACCESSOR;
-
+
private UnsafeUtil()
{
}
static {
- ByteOrder order = ByteOrder.nativeOrder();
- if (!order.equals(ByteOrder.LITTLE_ENDIAN)) {
- throw new SnappyError(SnappyErrorCode.UNSUPPORTED_PLATFORM, format("pure-java snappy requires a little endian platform (found %s)", order));
- }
try {
Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");
@@ -66,4 +59,6 @@ final class UnsafeUtil
throw new RuntimeException(e);
}
}
+
}
+
=====================================
src/main/resources/org/xerial/snappy/VERSION
=====================================
@@ -1,2 +1,2 @@
-SNAPPY_VERSION=1.1.7
+SNAPPY_VERSION=1.1.8
BITSHUFFLE_VERSION=0.3.2
=====================================
src/test/java/org/xerial/snappy/SnappyFramedStreamTest.java
=====================================
@@ -9,7 +9,6 @@ import static org.junit.Assert.fail;
import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG;
import static org.xerial.snappy.SnappyFramed.HEADER_BYTES;
import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG;
-import static org.xerial.snappy.SnappyFramed.maskedCrc32c;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -418,4 +417,9 @@ public class SnappyFramedStreamTest
assertEquals(random.length, length);
return random;
}
+
+ public static int maskedCrc32c(byte[] data)
+ {
+ return SnappyFramed.maskedCrc32c(new PureJavaCrc32C(), data, 0, data.length);
+ }
}
=====================================
src/test/resources/lib/Linux/libhadoop.so deleted
=====================================
Binary files a/src/test/resources/lib/Linux/libhadoop.so and /dev/null differ
=====================================
src/test/resources/lib/Linux/libsnappy.so deleted
=====================================
Binary files a/src/test/resources/lib/Linux/libsnappy.so and /dev/null differ
=====================================
src/test/resources/lib/MacOSX/libhadoop.dylib deleted
=====================================
Binary files a/src/test/resources/lib/MacOSX/libhadoop.dylib and /dev/null differ
=====================================
src/test/resources/lib/MacOSX/libsnappy.dylib deleted
=====================================
Binary files a/src/test/resources/lib/MacOSX/libsnappy.dylib and /dev/null differ
=====================================
version.sbt
=====================================
@@ -1 +1 @@
-version in ThisBuild := "1.1.7.7"
+version in ThisBuild := "1.1.8.3"
View it on GitLab: https://salsa.debian.org/java-team/snappy-java/-/compare/3ccb1f42b17e5e195d05c027a9bd07324500aac6...0abd6862f3399d2bcb0bce88a81015f24e19a797
--
View it on GitLab: https://salsa.debian.org/java-team/snappy-java/-/compare/3ccb1f42b17e5e195d05c027a9bd07324500aac6...0abd6862f3399d2bcb0bce88a81015f24e19a797
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-java-commits/attachments/20210125/9daca854/attachment.html>
More information about the pkg-java-commits
mailing list