[Git][java-team/snappy-java][upstream] New upstream version 1.1.7.1+dfsg
Andreas Tille
gitlab at salsa.debian.org
Wed May 2 21:30:38 BST 2018
Andreas Tille pushed to branch upstream at Debian Java Maintainers / snappy-java
Commits:
d09f5843 by Andreas Tille at 2018-05-02T22:15:21+02:00
New upstream version 1.1.7.1+dfsg
- - - - -
26 changed files:
- − .gitignore
- + .scalafmt.conf
- Makefile
- Makefile.common
- Milestone.md
- README.md
- build.sbt
- docker/Dockerfile.centos5-linux-x86_64-pic
- + docker/Makefile
- project/build.properties
- project/plugins.sbt
- sbt
- + script/travis-deploy.sh
- src/main/java/org/xerial/snappy/OSInfo.java
- + src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java
- src/main/java/org/xerial/snappy/SnappyOutputStream.java
- src/main/resources/org/xerial/snappy/VERSION
- + src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java
- + src/test/resources/lib/Linux/libhadoop.so
- + src/test/resources/lib/Linux/libsnappy.so
- + src/test/resources/lib/MacOSX/libhadoop.dylib
- + src/test/resources/lib/MacOSX/libsnappy.dylib
- + src/test/resources/log4j.properties
- src/test/scala/org/xerial/snappy/SnappyPerformanceTest.scala
- src/test/scala/org/xerial/snappy/SnappySpec.scala
- version.sbt
Changes:
=====================================
.gitignore deleted
=====================================
--- a/.gitignore
+++ /dev/null
@@ -1,25 +0,0 @@
-*~
-*.class
-*.log
-
-# IntelliJ
-.idea
-.idea_modules
-*.iml
-
-.DS_Store
-
-# sbt specific
-bin/.lib
-dist/*
-target/
-lib_managed/
-src_managed/
-project/boot/
-project/plugins/project/
-
-# Scala-IDE specific
-.scala_dependencies
-atlassian-ide-plugin.xml
-
-
=====================================
.scalafmt.conf
=====================================
--- /dev/null
+++ b/.scalafmt.conf
@@ -0,0 +1,3 @@
+maxColumn = 180
+style = defaultWithAlign
+optIn.breaksInsideChains = true
=====================================
Makefile
=====================================
--- a/Makefile
+++ b/Makefile
@@ -12,10 +12,10 @@ SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy.cc
SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(SNAPPY_VERSION)
SNAPPY_SRC:=$(addprefix $(SNAPPY_SRC_DIR)/,$(SNAPPY_CC))
SNAPPY_GIT_REPO_URL:=https://github.com/google/snappy
-SNAPPY_GIT_REV:=2d99bd14d471664758e4dfdf81b44f413a7353fd # 1.1.4
+SNAPPY_GIT_REV:=b02bfa754ebf27921d8da3bd2517eab445b84ff9 # 1.1.7
SNAPPY_UNPACKED:=$(TARGET)/snappy-extracted.log
SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
-SNAPPY_SOURCE_CONFIGURED:=$(TARGET)/snappy-configure.log
+SNAPPY_CMAKE_CACHE=$(SNAPPY_OUT)/CMakeCache.txt
BITSHUFFLE_ARCHIVE:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION).tar.gz
BITSHUFFLE_C:=bitshuffle_core.c iochain.c
@@ -39,7 +39,7 @@ $(SNAPPY_OUT)/%.o: $(BITSHUFFLE_SRC_DIR)/%.c
SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) $(patsubst %.c,%.o,$(BITSHUFFLE_C)) SnappyNative.o BitShuffleNative.o)
-CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR) -I$(BITSHUFFLE_SRC_DIR)
+CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR) -I$(SNAPPY_OUT) -I$(BITSHUFFLE_SRC_DIR)
ifndef CXXFLAGS_BITSHUFFLE
ifeq ($(OS_NAME)-$(OS_ARCH),Linux-x86_64)
@@ -70,20 +70,23 @@ $(SNAPPY_ARCHIVE):
$(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE)
$(TAR) xvfz $< -C $(TARGET)
touch $@
- cd $(SNAPPY_SRC_DIR) && ./configure
$(SNAPPY_GIT_UNPACKED):
+ @mkdir -p $(SNAPPY_OUT)
rm -rf $(SNAPPY_SRC_DIR)
@mkdir -p $(SNAPPY_SRC_DIR)
git clone $(SNAPPY_GIT_REPO_URL) $(SNAPPY_SRC_DIR)
git --git-dir=$(SNAPPY_SRC_DIR)/.git --work-tree=$(SNAPPY_SRC_DIR) checkout -b local/snappy-$(SNAPPY_VERSION) $(SNAPPY_GIT_REV)
touch $@
-$(SNAPPY_SOURCE_CONFIGURED): $(SNAPPY_GIT_UNPACKED)
- cd $(SNAPPY_SRC_DIR) && ./autogen.sh && ./configure
+$(SNAPPY_CMAKE_CACHE): $(SNAPPY_GIT_UNPACKED)
+ @mkdir -p $(SNAPPY_OUT)
+ cd $(SNAPPY_OUT) && cmake $(SNAPPY_CMAKE_OPTS) ../../$(SNAPPY_SRC_DIR)
touch $@
-jni-header: $(SNAPPY_SOURCE_CONFIGURED) $(BITSHUFFLE_UNPACKED) $(SRC)/org/xerial/snappy/SnappyNative.h $(SRC)/org/xerial/snappy/BitShuffleNative.h
+jni-header: $(SNAPPY_GIT_UNPACKED) $(BITSHUFFLE_UNPACKED) $(SRC)/org/xerial/snappy/SnappyNative.h $(SRC)/org/xerial/snappy/BitShuffleNative.h
+
+snappy-header: $(SNAPPY_CMAKE_CACHE)
$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class: $(SRC)/org/xerial/snappy/SnappyNative.java
@mkdir -p $(TARGET)/jni-classes
@@ -101,13 +104,18 @@ $(SRC)/org/xerial/snappy/BitShuffleNative.h: $(TARGET)/jni-classes/org/xerial/sn
$(SNAPPY_SRC): $(SNAPPY_GIT_UNPACKED)
+# aarch64 can use big-endian optimzied code
+ifeq ($(OS_ARCH),aarch64)
+SNAPPY_CXX_OPTS:=-DSNAPPY_IS_BIG_ENDIAN
+endif
+
$(SNAPPY_OUT)/%.o: $(SNAPPY_SRC_DIR)/%.cc
@mkdir -p $(@D)
- $(CXX) $(CXXFLAGS) -c $< -o $@
+ $(CXX) $(SNAPPY_CXX_OPTS) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/SnappyNative.o: $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h
@mkdir -p $(@D)
- $(CXX) $(CXXFLAGS) -c $< -o $@
+ $(CXX) $(SNAPPY_CXX_OPTS) $(CXXFLAGS) -c $< -o $@
$(SNAPPY_OUT)/BitShuffleNative.o: $(SRC)/org/xerial/snappy/BitShuffleNative.cpp $(SRC)/org/xerial/snappy/BitShuffleNative.h
@mkdir -p $(@D)
@@ -132,12 +140,13 @@ NATIVE_DLL:=$(NATIVE_DIR)/$(LIBNAME)
snappy-jar-version:=snappy-java-$(shell perl -npe "s/version in ThisBuild\s+:=\s+\"(.*)\"/\1/" version.sbt | sed -e "/^$$/d")
-native: jni-header $(NATIVE_DLL)
+native: jni-header snappy-header $(NATIVE_DLL)
+native-nocmake: jni-header $(NATIVE_DLL)
snappy: native $(TARGET)/$(snappy-jar-version).jar
native-all: win32 win64 mac64 native-arm linux32 linux64 linux-ppc64le linux-aarch64
-$(NATIVE_DLL): $(SNAPPY_SOURCE_CONFIGURED) $(SNAPPY_OUT)/$(LIBNAME)
+$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
@mkdir -p $(@D)
cp $(SNAPPY_OUT)/$(LIBNAME) $@
@mkdir -p $(NATIVE_TARGET_DIR)
@@ -154,10 +163,10 @@ test: $(NATIVE_DLL)
DOCKER_RUN_OPTS:=--rm
win32: jni-header
- ./docker/dockcross-windows-x86 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=i686-w64-mingw32.static- OS_NAME=Windows OS_ARCH=x86'
+ ./docker/dockcross-windows-x86 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native snappy-header native CROSS_PREFIX=i686-w64-mingw32.static- OS_NAME=Windows OS_ARCH=x86 SNAPPY_CMAKE_OPTS="-DHAVE_SYS_UIO_H=0"'
win64: jni-header
- ./docker/dockcross-windows-x64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=x86_64-w64-mingw32.static- OS_NAME=Windows OS_ARCH=x86_64'
+ ./docker/dockcross-windows-x64 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native snappy-header native CROSS_PREFIX=x86_64-w64-mingw32.static- OS_NAME=Windows OS_ARCH=x86_64 SNAPPY_CMAKE_OPTS="-DHAVE_SYS_UIO_H=0"'
# deprecated
mac32: jni-header
@@ -167,10 +176,10 @@ mac64: jni-header
docker run -it $(DOCKER_RUN_OPTS) -v $$PWD:/workdir -e CROSS_TRIPLE=x86_64-apple-darwin multiarch/crossbuild make clean-native native OS_NAME=Mac OS_ARCH=x86_64
linux32: jni-header
- docker run $(DOCKER_RUN_OPTS) -ti -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native OS_NAME=Linux OS_ARCH=x86'
+ docker run $(DOCKER_RUN_OPTS) -ti -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86'
linux64: jni-header
- docker run $(DOCKER_RUN_OPTS) -ti -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native OS_NAME=Linux OS_ARCH=x86_64'
+ docker run $(DOCKER_RUN_OPTS) -ti -v $$PWD:/work xerial/centos5-linux-x86_64-pic bash -c 'make clean-native native-nocmake OS_NAME=Linux OS_ARCH=x86_64'
freebsd64:
$(MAKE) native OS_NAME=FreeBSD OS_ARCH=x86_64
=====================================
Makefile.common
=====================================
--- a/Makefile.common
+++ b/Makefile.common
@@ -97,9 +97,9 @@ Linux-x86_64_SNAPPY_FLAGS :=
Linux-ppc_CXX := g++
Linux-ppc_STRIP := strip
ifeq ($(IBM_JDK_7),)
- Linux-ppc_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m32
+ Linux-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m32
else
- Linux-ppc_CXXFLAGS := -DHAVE_CONFIG_H -include lib/inc_linux/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m32
+ Linux-ppc_CXXFLAGS := -include lib/inc_linux/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m32
endif
Linux-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc_LIBNAME := libsnappyjava.so
@@ -108,9 +108,9 @@ Linux-ppc_SNAPPY_FLAGS :=
Linux-ppc64le_CXX := $(CROSS_PREFIX)g++
Linux-ppc64le_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
- Linux-ppc64le_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
+ Linux-ppc64le_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
- Linux-ppc64le_CXXFLAGS := -DHAVE_CONFIG_H -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
+ Linux-ppc64le_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
endif
Linux-ppc64le_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc64le_LIBNAME := libsnappyjava.so
@@ -119,9 +119,9 @@ Linux-ppc64le_SNAPPY_FLAGS :=
Linux-ppc64_CXX := $(CROSS_PREFIX)g++
Linux-ppc64_STRIP := $(CROSS_PREFIX)strip
ifeq ($(IBM_JDK_7),)
- Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
+ Linux-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
- Linux-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
+ Linux-ppc64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
endif
Linux-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-ppc64_LIBNAME := libsnappyjava.so
@@ -131,9 +131,9 @@ AIX-ppc_CXX := g++
AIX-ppc_STRIP := strip
AIX-ppc_LIBNAME := libsnappyjava.a
ifeq ($(IBM_JDK_7),)
- AIX-ppc_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix32
+ AIX-ppc_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix32
else
- AIX-ppc_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix32
+ AIX-ppc_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix32
endif
AIX-ppc_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc_SNAPPY_FLAGS :=
@@ -142,9 +142,9 @@ AIX-ppc64_CXX := g++
AIX-ppc64_STRIP := strip -X64
AIX-ppc64_LIBNAME := libsnappyjava.a
ifeq ($(IBM_JDK_7),)
- AIX-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix64
+ AIX-ppc64_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -maix64
else
- AIX-ppc64_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64
+ AIX-ppc64_CXXFLAGS := -I$(JAVA_HOME)/include/aix -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -maix64
endif
AIX-ppc64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ -lcrypt
AIX-ppc64_SNAPPY_FLAGS :=
@@ -152,9 +152,9 @@ AIX-ppc64_SNAPPY_FLAGS :=
Linux-s390_CXX := g++
Linux-s390_STRIP := strip
ifeq ($(IBM_JDK_7),)
- Linux-s390_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m31
+ Linux-s390_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m31
else
- Linux-s390_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m31
+ Linux-s390_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m31
endif
Linux-s390_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390_LIBNAME := libsnappyjava.so
@@ -163,9 +163,9 @@ Linux-s390_SNAPPY_FLAGS :=
Linux-s390x_CXX := g++
Linux-s390x_STRIP := strip
ifeq ($(IBM_JDK_7),)
- Linux-s390x_CXXFLAGS := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
+ Linux-s390x_CXXFLAGS := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
else
- Linux-s390x_CXXFLAGS := -DHAVE_CONFIG_H -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m64
+ Linux-s390x_CXXFLAGS := -I$(JAVA_HOME)/include/linux -Ilib/inc_ibm -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -m64
endif
Linux-s390x_LINKFLAGS := -shared -static-libgcc -static-libstdc++
Linux-s390x_LIBNAME := libsnappyjava.so
@@ -277,7 +277,7 @@ SNAPPY_FLAGS := $($(os_arch)_SNAPPY_FLAGS)
-CXXFLAGS := $(CXXFLAGS) -Ilib/include
+CXXFLAGS := $(CXXFLAGS) -DHAVE_CONFIG_H
ifneq ($(jni_include),)
CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)"
endif
=====================================
Milestone.md
=====================================
--- a/Milestone.md
+++ b/Milestone.md
@@ -1,9 +1,24 @@
Since version 1.1.0.x, Java 6 (1.6) or higher is required.
+## snappy-java-1.1.7.1 (2017-12-07)
+ * Fix for Android. No need to upgrade to this version if you are not using Android
+
+## snappy-java-1.1.7 (2017-11-30)
+ * Upgrade to [snappy-1.1.7](https://github.com/google/snappy/releases/tag/1.1.7)
+ * Improved performance for big-endian architecture
+ * The other performance improvement in [snappy-1.1.5](https://github.com/google/snappy/releases/tag/1.1.5)
+ * (internal) Changed to use docker + cmake for building native libraries
+ * Fix android architecture resolution
+ * Add hadoop compatible SnappyHadoopCompatibleOutputStream
+
## snappy-java-1.1.4 (2017-05-22)
* Upgrade to [snappy-1.1.4](https://github.com/google/snappy/releases/tag/1.1.4)
* Improved performance compression (5%), decompression (20%)
- * Added BitShuffle API for fast and better floating-point value compresssion
+ * Added BitShuffle API for fast and better integer and floating-point value compresssion
+ * Added native libraries for s390x, AIX/Linux ppc64/ppc64le
+ * Added prelimirary support Linux armv5, armv6, armv7, android-arm, aarch64
+ * Using docker for cross-compiling native libraries for various platforms
+ * Note: snappy-java for FreeBSD, SunOS are still using snappy 1.1.2. Your contributions of native libraries are welcome. Please send a pull request that contains the changes after running `make native test` in your platform.
## snappy-java-1.1.4-M3 (2017-02-16)
* Fix native libraries for ppc64 (bigendian) and ppc64le (little endian)
=====================================
README.md
=====================================
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-snappy-java [](https://travis-ci.org/xerial/snappy-java) [](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) [](http://www.javadoc.io/doc/org.xerial.snappy/snappy-java) [](https://www.versioneye.com/java/org.xerial.snappy:snappy-java/references)
+snappy-java [](https://travis-ci.org/xerial/snappy-java) [](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) [](http://javadoc-badge.appspot.com/org.xerial.snappy/snappy-java) [](https://www.versioneye.com/java/org.xerial.snappy:snappy-java/references)
===
snappy-java is a Java port of the snappy
<http://code.google.com/p/snappy/>, a fast C++ compresser/decompresser developed by Google.
@@ -28,7 +28,7 @@ snappy-java is a Java port of the snappy
## Download
- [](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) [](http://www.javadoc.io/doc/org.xerial.snappy/snappy-java)
+ [](https://maven-badges.herokuapp.com/maven-central/org.xerial.snappy/snappy-java/) [](http://javadoc-badge.appspot.com/org.xerial.snappy/snappy-java)
* [Release Notes](Milestone.md)
@@ -84,12 +84,14 @@ Stream-based compressor/decompressor `SnappyOutputStream`/`SnappyInputStream` ar
#### Compatibility Notes
* `SnappyOutputStream` and `SnappyInputStream` use `[magic header:16 bytes]([block size:int32][compressed data:byte array])*` format. You can read the result of `Snappy.compress` with `SnappyInputStream`, but you cannot read the compressed data generated by `SnappyOutputStream` with `Snappy.uncompress`. Here is the data format compatibility matrix:
-
-| Write\Read | `Snappy.uncompress` | `SnappyInputStream` | `SnappyFramedInputStream` |
-| --------------- |:-------------------:|:------------------:|:-----------------------:|
-| `Snappy.compress` | ok | ok | x |
-| `SnappyOutputStream` | x | ok | x |
-| `SnappyFramedOutputStream` | x | x | ok |
+ * `SnappyHadoopCompatibleOutputStream` does not emit a file header but write out the current block size as a preemble to each block
+
+| Write\Read | `Snappy.uncompress` | `SnappyInputStream` | `SnappyFramedInputStream` | `org.apache.hadoop.io.compress.SnappyCodec` |
+| --------------- |:-------------------:|:------------------:|:-----------------------:|:-------------------------------------------:|
+| `Snappy.compress` | ok | ok | x | x |
+| `SnappyOutputStream` | x | ok | x | x |
+| `SnappyFramedOutputStream` | x | x | ok | x |
+| `SnappyHadoopCompatibleOutputStream` | x | x | x | ok |
### BitShuffle API (Since 1.1.3-M2)
=====================================
build.sbt
=====================================
--- a/build.sbt
+++ b/build.sbt
@@ -1,26 +1,27 @@
-import de.johoop.findbugs4sbt.ReportType
-
name := "snappy-java"
-
-organization := "org.xerial.snappy"
-
+organization := "org.xerial.snappy"
organizationName := "xerial.org"
-
-description := "snappy-java: A fast compression/decompression library"
-
+description := "snappy-java: A fast compression/decompression library"
sonatypeProfileName := "org.xerial"
credentials ++= {
- if(sys.env.contains("SONATYPE_USERNAME") && sys.env.contains("SONATYPE_PASSWORD")) {
+ if (sys.env.contains("SONATYPE_USERNAME") && sys.env.contains("SONATYPE_PASSWORD")) {
Seq(Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", sys.env("SONATYPE_USERNAME"), sys.env("SONATYPE_PASSWORD")))
- }
- else {
+ } else {
Seq.empty
}
}
+publishTo := Some(
+ if (isSnapshot.value) {
+ Opts.resolver.sonatypeSnapshots
+ } else {
+ Opts.resolver.sonatypeStaging
+ }
+)
+
pomExtra := {
- <url>https://github.com/xerial/snappy-java</url>
+ <url>https://github.com/xerial/snappy-java</url>
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
@@ -54,92 +55,99 @@ pomExtra := {
</scm>
}
-scalaVersion in ThisBuild := "2.11.8"
+scalaVersion in ThisBuild := "2.12.4"
javacOptions in (Compile, compile) ++= Seq("-encoding", "UTF-8", "-Xlint:unchecked", "-Xlint:deprecation", "-source", "1.7", "-target", "1.7")
javacOptions in doc := {
- val opts = Seq("-source", "1.6")
- if (scala.util.Properties.isJavaAtLeast("1.8"))
- opts ++ Seq("-Xdoclint:none")
- else
- opts
+ val opts = Seq("-source", "1.6")
+ if (scala.util.Properties.isJavaAtLeast("1.8"))
+ opts ++ Seq("-Xdoclint:none")
+ else
+ opts
}
-testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v")
+// Configuration for SnappyHadoopCompatibleOutputStream testing
+fork in Test := true
+import java.io.File
+val libTemp = {
+ val path = s"${System.getProperty("java.io.tmpdir")}/snappy_test_${System.currentTimeMillis()}"
+ // certain older Linux systems (debian/trusty in Travis CI) requires the libsnappy.so, loaded by
+ // libhadoop.so, be copied to the temp path before the child JVM is forked.
+ // because of that, cannot define as an additional task in Test scope
+ IO.copyFile(file("src/test/resources/lib/Linux/libsnappy.so"), file(s"$path/libsnappy.so"))
+ IO.copyFile(file("src/test/resources/lib/Linux/libsnappy.so"), file(s"$path/libsnappy.so.1"))
+ path
+}
-concurrentRestrictions in Global := Seq(Tags.limit(Tags.Test, 1))
+val macOSXLibPath = s"$libTemp:${System.getenv("DYLD_LIBRARY_PATH")}"
+val linuxLibPath = s"$libTemp:${System.getenv("LD_LIBRARY_PATH")}"
-autoScalaLibrary := false
+// have to add to system dynamic library path since hadoop native library indirectly load libsnappy.1
+// can't use javaOptions in Test because it causes the expression to eval twice yielding different temp path values
+envVars in Test := Map("XERIAL_SNAPPY_LIB" -> libTemp, "DYLD_LIBRARY_PATH" -> macOSXLibPath, "LD_LIBRARY_PATH" -> linuxLibPath)
+testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v")
+concurrentRestrictions in Global := Seq(Tags.limit(Tags.Test, 1))
+autoScalaLibrary := false
crossPaths := false
-
logBuffered in Test := false
-incOptions := incOptions.value.withNameHashing(true)
-
-findbugsSettings
-
-findbugsReportType := Some(ReportType.FancyHtml)
-
+findbugsReportType := Some(FindbugsReport.FancyHtml)
findbugsReportPath := Some(crossTarget.value / "findbugs" / "report.html")
-jacoco.settings
-
libraryDependencies ++= Seq(
- "junit" % "junit" % "4.8.2" % "test",
- "org.codehaus.plexus" % "plexus-classworlds" % "2.4" % "test",
- "org.xerial.java" % "xerial-core" % "2.1" % "test",
- "org.xerial" % "xerial-core" % "3.2.3" % "test",
- "org.scalatest" % "scalatest_2.11" % "2.2.0" % "test",
- "org.osgi" % "org.osgi.core" % "4.3.0" % "provided",
- "com.novocode" % "junit-interface" % "0.10" % "test"
+ "junit" % "junit" % "4.8.2" % "test",
+ "org.codehaus.plexus" % "plexus-classworlds" % "2.4" % "test",
+ "org.xerial.java" % "xerial-core" % "2.1" % "test",
+ "org.wvlet.airframe" %% "airframe-log" % "0.25" % "test",
+ "org.scalatest" %% "scalatest" % "3.0.4" % "test",
+ "org.osgi" % "org.osgi.core" % "4.3.0" % "provided",
+ "com.novocode" % "junit-interface" % "0.11" % "test",
+ "org.apache.hadoop" % "hadoop-common" % "2.7.3" % "test" exclude ("org.xerial.snappy", "snappy-java")
)
-osgiSettings
-
+enablePlugins(SbtOsgi)
OsgiKeys.exportPackage := Seq("org.xerial.snappy", "org.xerial.snappy.buffer")
-
OsgiKeys.bundleSymbolicName := "org.xerial.snappy.snappy-java"
-
OsgiKeys.bundleActivator := Option("org.xerial.snappy.SnappyBundleActivator")
-
OsgiKeys.importPackage := Seq("""org.osgi.framework;version="[1.5,2)"""")
OsgiKeys.additionalHeaders := Map(
"Bundle-NativeCode" -> Seq(
-"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x86-64",
-"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x64",
-"org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=amd64",
-"org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86",
-"org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86",
-"org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64",
-"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64",
-"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x64",
-"org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=amd64",
-"org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86",
-"org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so;osname=linux;processor=aarch64",
-"org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm",
-"org/xerial/snappy/native/Linux/arm7/libsnappyjava.so;osname=linux;processor=arm_le",
-"org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64le",
-"org/xerial/snappy/native/Linux/s390x/libsnappyjava.so;osname=linux;processor=s390x",
-"org/xerial/snappy/native/AIX/ppc/libsnappyjava.a;osname=aix;processor=ppc",
-"org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64",
-"org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86",
-"org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64",
-"org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so;osname=sunos;processor=sparc"
-).mkString(","),
- "Bundle-DocURL" -> "http://www.xerial.org/",
- "Bundle-License" -> "http://www.apache.org/licenses/LICENSE-2.0.txt",
- "Bundle-ActivationPolicy" -> "lazy",
- "Bundle-Name" -> "snappy-java: A fast compression/decompression library"
+ "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x86-64",
+ "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=x64",
+ "org/xerial/snappy/native/Windows/x86_64/snappyjava.dll;osname=win32;processor=amd64",
+ "org/xerial/snappy/native/Windows/x86/snappyjava.dll;osname=win32;processor=x86",
+ "org/xerial/snappy/native/Mac/x86/libsnappyjava.jnilib;osname=macosx;processor=x86",
+ "org/xerial/snappy/native/Mac/x86_64/libsnappyjava.jnilib;osname=macosx;processor=x86-64",
+ "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x86-64",
+ "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=x64",
+ "org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so;osname=linux;processor=amd64",
+ "org/xerial/snappy/native/Linux/x86/libsnappyjava.so;osname=linux;processor=x86",
+ "org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so;osname=linux;processor=aarch64",
+ "org/xerial/snappy/native/Linux/arm/libsnappyjava.so;osname=linux;processor=arm",
+ "org/xerial/snappy/native/Linux/arm7/libsnappyjava.so;osname=linux;processor=arm_le",
+ "org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so;osname=linux;processor=ppc64le",
+ "org/xerial/snappy/native/Linux/s390x/libsnappyjava.so;osname=linux;processor=s390x",
+ "org/xerial/snappy/native/AIX/ppc/libsnappyjava.a;osname=aix;processor=ppc",
+ "org/xerial/snappy/native/AIX/ppc64/libsnappyjava.a;osname=aix;processor=ppc64",
+ "org/xerial/snappy/native/SunOS/x86/libsnappyjava.so;osname=sunos;processor=x86",
+ "org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so;osname=sunos;processor=x86-64",
+ "org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so;osname=sunos;processor=sparc"
+ ).mkString(","),
+ "Bundle-DocURL" -> "http://www.xerial.org/",
+ "Bundle-License" -> "http://www.apache.org/licenses/LICENSE-2.0.txt",
+ "Bundle-ActivationPolicy" -> "lazy",
+ "Bundle-Name" -> "snappy-java: A fast compression/decompression library"
)
import ReleaseTransformations._
import sbtrelease._
releaseTagName := { (version in ThisBuild).value }
+releasePublishArtifactsAction := PgpKeys.publishSigned.value
releaseProcess := Seq[ReleaseStep](
checkSnapshotDependencies,
@@ -149,14 +157,9 @@ releaseProcess := Seq[ReleaseStep](
setReleaseVersion,
commitReleaseVersion,
tagRelease,
- ReleaseStep(action = Command.process("publishSigned", _)),
+ publishArtifacts,
setNextVersion,
commitNextVersion,
- ReleaseStep(action = Command.process("sonatypeReleaseAll", _)),
+ releaseStepCommand("sonatypeReleaseAll"),
pushChanges
)
-
-
-com.etsy.sbt.Checkstyle.checkstyleSettings
-
-com.etsy.sbt.Checkstyle.CheckstyleTasks.checkstyleConfig := file("src/checkstyle/checks.xml")
=====================================
docker/Dockerfile.centos5-linux-x86_64-pic
=====================================
--- a/docker/Dockerfile.centos5-linux-x86_64-pic
+++ b/docker/Dockerfile.centos5-linux-x86_64-pic
@@ -1,6 +1,10 @@
FROM centos:5
MAINTAINER Taro L. Saito <leo at xerial.org>
+RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+RUN sed -i 's/mirrorlist/#mirrorlist/' /etc/yum.repos.d/*.repo
+RUN sed -i 's|#baseurl=http://mirror.centos.org/centos/$releasever|baseurl=http://vault.centos.org/5.11|' /etc/yum.repos.d/*.repo
+
RUN yum -y install make gcc gcc-c++ glibc-devel perl wget bzip2 curl \
&& rm -rf /var/lib/apt/lists/*
@@ -21,6 +25,16 @@ RUN cd /tmp/work/objdir \
&& make install \
&& rm -rf /tmp/work
+#RUN mkdir /tmp/cmake \
+# && cd /tmp/cmake \
+# && wget --no-check-certificate https://cmake.org/files/v3.10/cmake-3.10.0.tar.gz \
+# && tar xvfz cmake-3.10.0.tar.gz \
+# && cd cmake-3.10.0 \
+# && CXX=/usr/local/gcc-4.8.3/bin/g++ ./bootstrap \
+# && make \
+# && make install \
+# && rm -rf /tmp/cmake
+
ENV PATH /usr/local/gcc-4.8.3/bin:$PATH
ENV LD_LIBRARY_PATH /usr/local/gcc-4.8.3/lib64/:$LD_LIBRARY_PATH
=====================================
docker/Makefile
=====================================
--- /dev/null
+++ b/docker/Makefile
@@ -0,0 +1,6 @@
+
+centos5-image:
+ docker build -f Dockerfile.centos5-linux-x86_64-pic -t xerial/centos5-linux-x86_64:latest
+ # dockerhub login
+ # docker login --username=xerial
+ # docker push xerial/centos5-linux-x86_64:latest
=====================================
project/build.properties
=====================================
--- a/project/build.properties
+++ b/project/build.properties
@@ -1,2 +1,2 @@
-sbt.version=0.13.13
+sbt.version=1.0.3
=====================================
project/plugins.sbt
=====================================
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,8 +1,8 @@
-addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.3")
-addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
-addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
-addSbtPlugin("de.johoop" % "findbugs4sbt" % "1.4.0")
-addSbtPlugin("de.johoop" % "jacoco4sbt" % "2.1.5")
-addSbtPlugin("com.typesafe.sbt" % "sbt-osgi" % "0.7.0")
-addSbtPlugin("com.etsy" % "sbt-checkstyle-plugin" % "0.4.3")
-addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M15")
+addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.6")
+addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.0")
+addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0")
+addSbtPlugin("com.github.sbt" % "sbt-findbugs" % "2.0.0")
+addSbtPlugin("com.github.sbt" % "sbt-jacoco" % "3.0.3")
+addSbtPlugin("com.typesafe.sbt" % "sbt-osgi" % "0.9.2")
+addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC13")
+addSbtPlugin("com.geirsson" % "sbt-scalafmt" % "1.3.0")
=====================================
sbt
=====================================
--- a/sbt
+++ b/sbt
@@ -3,59 +3,62 @@
# A more capable sbt runner, coincidentally also called sbt.
# Author: Paul Phillips <paulp at improving.org>
-# todo - make this dynamic
-declare -r sbt_release_version="0.13.8"
-declare -r sbt_unreleased_version="0.13.9-M1"
+set -o pipefail
+
+declare -r sbt_release_version="0.13.16"
+declare -r sbt_unreleased_version="0.13.16"
+
+declare -r latest_213="2.13.0-M2"
+declare -r latest_212="2.12.4"
+declare -r latest_211="2.11.11"
+declare -r latest_210="2.10.6"
+declare -r latest_29="2.9.3"
+declare -r latest_28="2.8.2"
+
declare -r buildProps="project/build.properties"
-declare sbt_jar sbt_dir sbt_create sbt_version
-declare scala_version sbt_explicit_version
-declare verbose noshare batch trace_level log_level
-declare sbt_saved_stty debugUs
+declare -r sbt_launch_ivy_release_repo="http://repo.typesafe.com/typesafe/ivy-releases"
+declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots"
+declare -r sbt_launch_mvn_release_repo="http://repo.scala-sbt.org/scalasbt/maven-releases"
+declare -r sbt_launch_mvn_snapshot_repo="http://repo.scala-sbt.org/scalasbt/maven-snapshots"
-echoerr () { echo >&2 "$@"; }
-vlog () { [[ -n "$verbose" ]] && echoerr "$@"; }
+declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m"
+declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
-# spaces are possible, e.g. sbt.version = 0.13.0
-build_props_sbt () {
- [[ -r "$buildProps" ]] && \
- grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }'
-}
+declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new
+declare sbt_explicit_version
+declare verbose noshare batch trace_level
+declare debugUs
-update_build_props_sbt () {
- local ver="$1"
- local old="$(build_props_sbt)"
+declare java_cmd="java"
+declare sbt_launch_dir="$HOME/.sbt/launchers"
+declare sbt_launch_repo
- [[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && {
- perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps"
- grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps"
+# pull -J and -D options to give to java.
+declare -a java_args scalac_args sbt_commands residual_args
- vlog "!!!"
- vlog "!!! Updated file $buildProps setting sbt.version to: $ver"
- vlog "!!! Previous value was: $old"
- vlog "!!!"
- }
-}
+# args to jvm/sbt via files or environment variables
+declare -a extra_jvm_opts extra_sbt_opts
-set_sbt_version () {
- sbt_version="${sbt_explicit_version:-$(build_props_sbt)}"
- [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version
- export sbt_version
-}
+echoerr () { echo >&2 "$@"; }
+vlog () { [[ -n "$verbose" ]] && echoerr "$@"; }
+die () { echo "Aborting: $@" ; exit 1; }
-# restore stty settings (echo in particular)
-onSbtRunnerExit() {
- [[ -n "$sbt_saved_stty" ]] || return
- vlog ""
- vlog "restoring stty: $sbt_saved_stty"
- stty "$sbt_saved_stty"
- unset sbt_saved_stty
-}
+setTrapExit () {
+ # save stty and trap exit, to ensure echo is re-enabled if we are interrupted.
+ export SBT_STTY="$(stty -g 2>/dev/null)"
-# save stty and trap exit, to ensure echo is reenabled if we are interrupted.
-trap onSbtRunnerExit EXIT
-sbt_saved_stty="$(stty -g 2>/dev/null)"
-vlog "Saved stty: $sbt_saved_stty"
+ # restore stty settings (echo in particular)
+ onSbtRunnerExit() {
+ [ -t 0 ] || return
+ vlog ""
+ vlog "restoring stty: $SBT_STTY"
+ stty "$SBT_STTY"
+ }
+
+ vlog "saving stty: $SBT_STTY"
+ trap onSbtRunnerExit EXIT
+}
# this seems to cover the bases on OSX, and someone will
# have to tell me about the others.
@@ -71,21 +74,8 @@ get_script_path () {
fi
}
-die() {
- echo "Aborting: $@"
- exit 1
-}
-
-make_url () {
- version="$1"
-
- case "$version" in
- 0.7.*) echo "http://simple-build-tool.googlecode.com/files/sbt-launch-0.7.7.jar" ;;
- 0.10.* ) echo "$sbt_launch_repo/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
- 0.11.[12]) echo "$sbt_launch_repo/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
- *) echo "$sbt_launch_repo/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;;
- esac
-}
+declare -r script_path="$(get_script_path "$BASH_SOURCE")"
+declare -r script_name="${script_path##*/}"
init_default_option_file () {
local overriding_var="${!1}"
@@ -99,97 +89,116 @@ init_default_option_file () {
echo "$default_file"
}
-declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC"
-declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation"
-declare -r default_jvm_opts_common="-Xms512m -Xmx2048m -Xss2m $jit_opts $cms_opts"
-declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
-declare -r latest_28="2.8.2"
-declare -r latest_29="2.9.3"
-declare -r latest_210="2.10.5"
-declare -r latest_211="2.11.7"
-
-declare -r script_path="$(get_script_path "$BASH_SOURCE")"
-declare -r script_name="${script_path##*/}"
-
-# some non-read-onlies set with defaults
-declare java_cmd="java"
declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)"
declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)"
-declare sbt_launch_repo="http://repo.typesafe.com/typesafe/ivy-releases"
-# pull -J and -D options to give to java.
-declare -a residual_args
-declare -a java_args
-declare -a scalac_args
-declare -a sbt_commands
+build_props_sbt () {
+ [[ -r "$buildProps" ]] && \
+ grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }'
+}
-# args to jvm/sbt via files or environment variables
-declare -a extra_jvm_opts extra_sbt_opts
+update_build_props_sbt () {
+ local ver="$1"
+ local old="$(build_props_sbt)"
-addJava () {
- vlog "[addJava] arg = '$1'"
- java_args+=("$1")
+ [[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && {
+ perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps"
+ grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps"
+
+ vlog "!!!"
+ vlog "!!! Updated file $buildProps setting sbt.version to: $ver"
+ vlog "!!! Previous value was: $old"
+ vlog "!!!"
+ }
}
-addSbt () {
- vlog "[addSbt] arg = '$1'"
- sbt_commands+=("$1")
+
+set_sbt_version () {
+ sbt_version="${sbt_explicit_version:-$(build_props_sbt)}"
+ [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version
+ export sbt_version
+}
+
+url_base () {
+ local version="$1"
+
+ case "$version" in
+ 0.7.*) echo "http://simple-build-tool.googlecode.com" ;;
+ 0.10.* ) echo "$sbt_launch_ivy_release_repo" ;;
+ 0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;;
+ 0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss"
+ echo "$sbt_launch_ivy_snapshot_repo" ;;
+ 0.*) echo "$sbt_launch_ivy_release_repo" ;;
+ *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss"
+ echo "$sbt_launch_mvn_snapshot_repo" ;;
+ *) echo "$sbt_launch_mvn_release_repo" ;;
+ esac
+}
+
+make_url () {
+ local version="$1"
+
+ local base="${sbt_launch_repo:-$(url_base "$version")}"
+
+ case "$version" in
+ 0.7.*) echo "$base/files/sbt-launch-0.7.7.jar" ;;
+ 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
+ 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
+ 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;;
+ *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch.jar" ;;
+ esac
}
+
+addJava () { vlog "[addJava] arg = '$1'" ; java_args+=("$1"); }
+addSbt () { vlog "[addSbt] arg = '$1'" ; sbt_commands+=("$1"); }
+addScalac () { vlog "[addScalac] arg = '$1'" ; scalac_args+=("$1"); }
+addResidual () { vlog "[residual] arg = '$1'" ; residual_args+=("$1"); }
+
+addResolver () { addSbt "set resolvers += $1"; }
+addDebugger () { addJava "-Xdebug" ; addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; }
setThisBuild () {
vlog "[addBuild] args = '$@'"
local key="$1" && shift
addSbt "set $key in ThisBuild := $@"
}
-addScalac () {
- vlog "[addScalac] arg = '$1'"
- scalac_args+=("$1")
-}
-addResidual () {
- vlog "[residual] arg = '$1'"
- residual_args+=("$1")
-}
-addResolver () {
- addSbt "set resolvers += $1"
-}
-addDebugger () {
- addJava "-Xdebug"
- addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
-}
setScalaVersion () {
[[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")'
addSbt "++ $1"
}
setJavaHome () {
java_cmd="$1/bin/java"
- setThisBuild javaHome "Some(file(\"$1\"))"
+ setThisBuild javaHome "_root_.scala.Some(file(\"$1\"))"
export JAVA_HOME="$1"
export JDK_HOME="$1"
export PATH="$JAVA_HOME/bin:$PATH"
}
-setJavaHomeQuietly () {
- addSbt warn
- setJavaHome "$1"
- addSbt info
-}
-# if set, use JDK_HOME/JAVA_HOME over java found in path
-if [[ -e "$JDK_HOME/lib/tools.jar" ]]; then
- setJavaHomeQuietly "$JDK_HOME"
-elif [[ -e "$JAVA_HOME/bin/java" ]]; then
- setJavaHomeQuietly "$JAVA_HOME"
-fi
+getJavaVersion() { "$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d \"; }
-# directory to store sbt launchers
-declare sbt_launch_dir="$HOME/.sbt/launchers"
-[[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir"
-[[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)"
+checkJava() {
+ # Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME
+
+ [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java"
+ [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java"
+
+ if [[ -n "$java" ]]; then
+ pathJavaVersion=$(getJavaVersion java)
+ homeJavaVersion=$(getJavaVersion "$java")
+ if [[ "$pathJavaVersion" != "$homeJavaVersion" ]]; then
+ echoerr "Warning: Java version mismatch between PATH and JAVA_HOME/JDK_HOME, sbt will use the one in PATH"
+ echoerr " Either: fix your PATH, remove JAVA_HOME/JDK_HOME or use -java-home"
+ echoerr " java version from PATH: $pathJavaVersion"
+ echoerr " java version from JAVA_HOME/JDK_HOME: $homeJavaVersion"
+ fi
+ fi
+}
java_version () {
- local version=$("$java_cmd" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d \")
+ local version=$(getJavaVersion "$java_cmd")
vlog "Detected Java version: $version"
echo "${version:2:1}"
}
-# MaxPermSize critical on pre-8 jvms but incurs noisy warning on 8+
+# MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+
default_jvm_opts () {
local v="$(java_version)"
if [[ $v -ge 8 ]]; then
@@ -222,16 +231,23 @@ execRunner () {
vlog ""
}
- [[ -n "$batch" ]] && exec </dev/null
- exec "$@"
-}
+ setTrapExit
-jar_url () {
- make_url "$1"
+ if [[ -n "$batch" ]]; then
+ "$@" < /dev/null
+ else
+ "$@"
+ fi
}
+jar_url () { make_url "$1"; }
+
+is_cygwin () [[ "$(uname -a)" == "CYGWIN"* ]]
+
jar_file () {
- echo "$sbt_launch_dir/$1/sbt-launch.jar"
+ is_cygwin \
+ && echo "$(cygpath -w $sbt_launch_dir/"$1"/sbt-launch.jar)" \
+ || echo "$sbt_launch_dir/$1/sbt-launch.jar"
}
download_url () {
@@ -246,19 +262,26 @@ download_url () {
if which curl >/dev/null; then
curl --fail --silent --location "$url" --output "$jar"
elif which wget >/dev/null; then
- wget --quiet -O "$jar" "$url"
+ wget -q -O "$jar" "$url"
fi
} && [[ -r "$jar" ]]
}
acquire_sbt_jar () {
- sbt_url="$(jar_url "$sbt_version")"
- sbt_jar="$(jar_file "$sbt_version")"
-
- [[ -r "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar"
+ {
+ sbt_jar="$(jar_file "$sbt_version")"
+ [[ -r "$sbt_jar" ]]
+ } || {
+ sbt_jar="$HOME/.ivy2/local/org.scala-sbt/sbt-launch/$sbt_version/jars/sbt-launch.jar"
+ [[ -r "$sbt_jar" ]]
+ } || {
+ sbt_jar="$(jar_file "$sbt_version")"
+ download_url "$(make_url "$sbt_version")" "$sbt_jar"
+ }
}
usage () {
+ set_sbt_version
cat <<EOM
Usage: $script_name [options]
@@ -289,20 +312,23 @@ runner with the -x option.
-jvm-debug <port> Turn on JVM debugging, open at the given port.
-batch Disable interactive mode
-prompt <expr> Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted
+ -script <file> Run the specified file as a scala script
# sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version)
-sbt-force-latest force the use of the latest release of sbt: $sbt_release_version
-sbt-version <version> use the specified version of sbt (default: $sbt_release_version)
-sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version
-sbt-jar <path> use the specified jar as the sbt launcher
- -sbt-launch-dir <path> directory to hold sbt launchers (default: ~/.sbt/launchers)
- -sbt-launch-repo <url> repo url for downloading sbt launcher jar (default: $sbt_launch_repo)
+ -sbt-launch-dir <path> directory to hold sbt launchers (default: $sbt_launch_dir)
+ -sbt-launch-repo <url> repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version"))
# scala version (default: as chosen by sbt)
-28 use $latest_28
-29 use $latest_29
-210 use $latest_210
-211 use $latest_211
+ -212 use $latest_212
+ -213 use $latest_213
-scala-home <path> use the scala build at the specified directory
-scala-version <version> use the specified version of scala
-binary-version <version> use the specified scala version when searching for dependencies
@@ -329,8 +355,7 @@ runner with the -x option.
EOM
}
-process_args ()
-{
+process_args () {
require_arg () {
local type="$1"
local opt="$2"
@@ -342,11 +367,11 @@ process_args ()
}
while [[ $# -gt 0 ]]; do
case "$1" in
- -h|-help) usage; exit 1 ;;
+ -h|-help) usage; exit 0 ;;
-v) verbose=true && shift ;;
- -d) addSbt "--debug" && addSbt debug && shift ;;
- -w) addSbt "--warn" && addSbt warn && shift ;;
- -q) addSbt "--error" && addSbt error && shift ;;
+ -d) addSbt "--debug" && shift ;;
+ -w) addSbt "--warn" && shift ;;
+ -q) addSbt "--error" && shift ;;
-x) debugUs=true && shift ;;
-trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;;
-ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
@@ -355,10 +380,11 @@ process_args ()
-sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
-sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
-debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
- -offline) addSbt "set offline := true" && shift ;;
+ -offline) addSbt "set offline in Global := true" && shift ;;
-jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;;
-batch) batch=true && shift ;;
-prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;;
+ -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;;
-sbt-create) sbt_create=true && shift ;;
-sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
@@ -369,7 +395,7 @@ process_args ()
-sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;;
-scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;;
-binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;;
- -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "Some(file(\"$2\"))" && shift 2 ;;
+ -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;;
-java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;;
-sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;;
-jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;;
@@ -381,10 +407,9 @@ process_args ()
-29) setScalaVersion "$latest_29" && shift ;;
-210) setScalaVersion "$latest_210" && shift ;;
-211) setScalaVersion "$latest_211" && shift ;;
-
- --debug) addSbt debug && addResidual "$1" && shift ;;
- --warn) addSbt warn && addResidual "$1" && shift ;;
- --error) addSbt error && addResidual "$1" && shift ;;
+ -212) setScalaVersion "$latest_212" && shift ;;
+ -213) setScalaVersion "$latest_213" && shift ;;
+ new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;;
*) addResidual "$1" && shift ;;
esac
done
@@ -395,8 +420,10 @@ process_args "$@"
# skip #-styled comments and blank lines
readConfigFile() {
- while read line; do
- [[ $line =~ ^# ]] || [[ -z $line ]] || echo "$line"
+ local end=false
+ until $end; do
+ read || end=true
+ [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY"
done < "$1"
}
@@ -421,6 +448,8 @@ argumentCount=$#
# set sbt version
set_sbt_version
+checkJava
+
# only exists in 0.12+
setTraceLevel() {
case "$sbt_version" in
@@ -433,19 +462,21 @@ setTraceLevel() {
[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\""
# Update build.properties on disk to set explicit version - sbt gives us no choice
-[[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version"
+[[ -n "$sbt_explicit_version" && -z "$sbt_new" ]] && update_build_props_sbt "$sbt_explicit_version"
vlog "Detected sbt version $sbt_version"
-[[ -n "$scala_version" ]] && vlog "Overriding scala version to $scala_version"
-
-# no args - alert them there's stuff in here
-(( argumentCount > 0 )) || {
- vlog "Starting $script_name: invoke with -help for other options"
- residual_args=( shell )
-}
+if [[ -n "$sbt_script" ]]; then
+ residual_args=( $sbt_script ${residual_args[@]} )
+else
+ # no args - alert them there's stuff in here
+ (( argumentCount > 0 )) || {
+ vlog "Starting $script_name: invoke with -help for other options"
+ residual_args=( shell )
+ }
+fi
-# verify this is an sbt dir or -create was given
-[[ -r ./build.sbt || -d ./project || -n "$sbt_create" ]] || {
+# verify this is an sbt dir, -create was given or user attempts to run a scala script
+[[ -r ./build.sbt || -d ./project || -n "$sbt_create" || -n "$sbt_script" || -n "$sbt_new" ]] || {
cat <<EOM
$(pwd) doesn't appear to be an sbt project.
If you want to start sbt anyway, run:
@@ -458,6 +489,10 @@ EOM
# pick up completion if present; todo
[[ -r .sbt_completion.sh ]] && source .sbt_completion.sh
+# directory to store sbt launchers
+[[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir"
+[[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)"
+
# no jar? download it.
[[ -r "$sbt_jar" ]] || acquire_sbt_jar || {
# still no jar? uh-oh.
@@ -519,8 +554,8 @@ mainFiltered () {
echoLine () {
local line="$1"
- local line1="$(echo "$line" | sed -r 's/\r\x1BM\x1B\[2K//g')" # This strips the OverwriteLine code.
- local line2="$(echo "$line1" | sed -r 's/\x1B\[[0-9;]*[JKmsu]//g')" # This strips all codes - we test regexes against this.
+ local line1="$(echo "$line" | sed 's/\r\x1BM\x1B\[2K//g')" # This strips the OverwriteLine code.
+ local line2="$(echo "$line1" | sed 's/\x1B\[[0-9;]*[JKmsu]//g')" # This strips all codes - we test regexes against this.
if [[ $line2 =~ $excludeRegex ]]; then
[[ -n $debugUs ]] && echo "[X] $line1"
=====================================
script/travis-deploy.sh
=====================================
--- /dev/null
+++ b/script/travis-deploy.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ev
+
+VERSION=`perl -npe "s/version in ThisBuild\s+:=\s+\"(.*)\"/\1/" version.sbt | sed -e "/^$/d"`
+
+# Deploy a snapshot version only for master branch and jdk8
+if [[ "$TRAVIS_JDK_VERSION" == "oraclejdk8" ]]; then
+ if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] && [[ "$VERSION" == *SNAPSHOT ]]; then
+ ./sbt "; test; publish";
+ else
+ ./sbt test;
+ fi;
+else
+ ./sbt test;
+fi;
=====================================
src/main/java/org/xerial/snappy/OSInfo.java
=====================================
--- a/src/main/java/org/xerial/snappy/OSInfo.java
+++ b/src/main/java/org/xerial/snappy/OSInfo.java
@@ -151,11 +151,6 @@ public class OSInfo {
}
static String resolveArmArchType() {
- // For Android
- if(isAndroid()) {
- return "android-arm";
- }
-
if(System.getProperty("os.name").contains("Linux")) {
String armType = getHardwareName();
// armType (uname -m) can be armv5t, armv5te, armv5tej, armv5tejl, armv6, armv7, armv7l, i686
@@ -207,6 +202,11 @@ public class OSInfo {
public static String getArchName() {
String osArch = System.getProperty("os.arch");
+ // For Android
+ if(isAndroid()) {
+ return "android-arm";
+ }
+
if(osArch.startsWith("arm")) {
osArch = resolveArmArchType();
}
=====================================
src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java
=====================================
--- /dev/null
+++ b/src/main/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStream.java
@@ -0,0 +1,30 @@
+package org.xerial.snappy;
+
+import java.io.OutputStream;
+
+import org.xerial.snappy.buffer.CachedBufferAllocator;
+
+public class SnappyHadoopCompatibleOutputStream extends SnappyOutputStream
+{
+ public SnappyHadoopCompatibleOutputStream(OutputStream out)
+ {
+ this(out, DEFAULT_BLOCK_SIZE);
+ }
+
+ public SnappyHadoopCompatibleOutputStream(OutputStream out, int blockSize)
+ {
+ super(out, blockSize, CachedBufferAllocator.getBufferAllocatorFactory());
+ }
+
+ @Override
+ protected int writeHeader()
+ {
+ return 0;
+ }
+
+ @Override
+ protected void writeBlockPreemble()
+ {
+ writeCurrentDataSize();
+ }
+}
=====================================
src/main/java/org/xerial/snappy/SnappyOutputStream.java
=====================================
--- a/src/main/java/org/xerial/snappy/SnappyOutputStream.java
+++ b/src/main/java/org/xerial/snappy/SnappyOutputStream.java
@@ -72,6 +72,7 @@ public class SnappyOutputStream
protected byte[] outputBuffer;
private int inputCursor = 0;
private int outputCursor = 0;
+ private boolean headerWritten;
private boolean closed;
public SnappyOutputStream(OutputStream out)
@@ -101,8 +102,6 @@ public class SnappyOutputStream
inputBuffer = inputBufferAllocator.allocate(inputSize);
outputBuffer = outputBufferAllocator.allocate(outputSize);
-
- outputCursor = SnappyCodec.currentHeader.writeHeader(outputBuffer, 0);
}
/* (non-Javadoc)
@@ -369,10 +368,18 @@ public class SnappyOutputStream
return; // no need to dump
}
+ if (!headerWritten) {
+ outputCursor = writeHeader();
+ headerWritten = true;
+ }
+
// Compress and dump the buffer content
if (!hasSufficientOutputBufferFor(inputCursor)) {
dumpOutput();
}
+
+ writeBlockPreemble();
+
int compressedSize = Snappy.compress(inputBuffer, 0, inputCursor, outputBuffer, outputCursor + 4);
// Write compressed data size
writeInt(outputBuffer, outputCursor, compressedSize);
@@ -380,6 +387,24 @@ public class SnappyOutputStream
inputCursor = 0;
}
+ protected int writeHeader(){
+ return SnappyCodec.currentHeader.writeHeader(outputBuffer, 0);
+ }
+
+ /**
+ * Optionally write a preemble before a block. Hadoop requires the actual block data size being written. This base
+ * implementation does nothing. Derive implementation can call {@code writeCurrentDataSize()}.
+ */
+ protected void writeBlockPreemble()
+ {
+ // do nothing
+ }
+
+ protected void writeCurrentDataSize(){
+ writeInt(outputBuffer, outputCursor, inputCursor);
+ outputCursor += 4;
+ }
+
/**
* close the stream
*/
=====================================
src/main/resources/org/xerial/snappy/VERSION
=====================================
--- a/src/main/resources/org/xerial/snappy/VERSION
+++ b/src/main/resources/org/xerial/snappy/VERSION
@@ -1,2 +1,2 @@
-SNAPPY_VERSION=1.1.4
+SNAPPY_VERSION=1.1.7
BITSHUFFLE_VERSION=0.3.2
=====================================
src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java
=====================================
--- /dev/null
+++ b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java
@@ -0,0 +1,148 @@
+package org.xerial.snappy;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.SystemUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.SnappyCodec;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class SnappyHadoopCompatibleOutputStreamTest
+{
+ private static File tempNativeLibFolder;
+
+ @BeforeClass
+ public static void loadHadoopNativeLibrary() throws Exception
+ {
+ final String libResourceFolder;
+ Map<String, String> libraryNames = new LinkedHashMap<>();
+ if (SystemUtils.IS_OS_LINUX) {
+ libResourceFolder = "/lib/Linux";
+ libraryNames.put("libhadoop.so", "libhadoop.so");
+ // certain Linux systems need these shared library be copied before the JVM started, see build.sbt
+ libraryNames.put("libsnappy.so", "libsnappy.so");
+ libraryNames.put("libsnappy.so.1", "libsnappy.so");
+ } else if (SystemUtils.IS_OS_MAC_OSX) {
+ libResourceFolder = "/lib/MacOSX";
+ libraryNames.put("libhadoop.dylib", "libhadoop.dylib");
+ libraryNames.put("libsnappy.dylib", "libsnappy.dylib");
+ libraryNames.put("libsnappy.1.dylib", "libsnappy.dylib");
+ } else {
+ return; // not support
+ }
+
+ String testLibDir = System.getenv("XERIAL_SNAPPY_LIB");
+
+ tempNativeLibFolder = new File(testLibDir);
+ tempNativeLibFolder.mkdirs();
+
+ for (Map.Entry<String, String> entry : libraryNames.entrySet()) {
+ copyNativeLibraryToFS(libResourceFolder, entry.getValue(), entry.getKey());
+ }
+
+ System.setProperty("java.library.path", tempNativeLibFolder.getAbsolutePath());
+
+ // credit: https://stackoverflow.com/questions/15409223/adding-new-paths-for-native-libraries-at-runtime-in-java
+ //set sys_paths to null so that java.library.path will be reevalueted next time it is needed
+ final Field sysPathsField = ClassLoader.class.getDeclaredField("sys_paths");
+ sysPathsField.setAccessible(true);
+ sysPathsField.set(null, null);
+ }
+
+ private static void copyNativeLibraryToFS(String libResourceFolder, String libraryName, String toLibraryName) {
+ final String libraryResourceName = libResourceFolder + "/" + libraryName;
+ final File libraryPath = new File(tempNativeLibFolder, toLibraryName);
+ try (InputStream inputStream = SnappyHadoopCompatibleOutputStream.class.getResourceAsStream(libraryResourceName);
+ FileOutputStream outputStream = new FileOutputStream(libraryPath)) {
+ IOUtils.copy(inputStream, outputStream);
+ FileDescriptor fd = outputStream.getFD();
+ fd.sync();
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @AfterClass
+ public static void cleanUpLibraryFolder()
+ {
+ FileUtils.deleteQuietly(tempNativeLibFolder);
+ }
+
+ @Test
+ public void testXerialCompressionHadoopDecompressionCodec() throws Exception
+ {
+ String os = OSInfo.getOSName();
+ String arch = OSInfo.getArchName();
+ if(!((os.equals("Linux") || os.equals("Mac")) && arch.equals("x86_64"))) {
+ // This test works only in Linux/Mac x86_64
+ System.err.println("SnappyHadoopCompatibleOutputStreamTest works only in 64-bit Linux/Mac");
+ return;
+ }
+
+ File inputFile = File.createTempFile("TEST_hadoop_compatibility", ".txt");
+ File snappyFile = File.createTempFile("TEST_hadoop_compatibility", ".snappy");
+ InputStream snappyInput = null;
+ FileOutputStream outputStream = new FileOutputStream(inputFile);
+ try {
+ String text = "";
+ for (int i = 0; i < 1024; i++) {
+ text += "Some long long strings to be compressed. Some long long strings to be compressed.";
+ }
+ text += "odd bytes";
+ final byte[] bytes = text.getBytes("UTF-8");
+ outputStream.write(bytes);
+ outputStream.flush();
+ outputStream.close();
+
+ compress(inputFile, snappyFile);
+
+ // Test using Hadoop's Snappy Codec
+ if (tempNativeLibFolder != null) {
+ SnappyCodec hadoopCodec = new SnappyCodec();
+ hadoopCodec.setConf(new Configuration());
+ snappyInput = hadoopCodec.createInputStream(new FileInputStream(snappyFile));
+ byte[] buf = new byte[bytes.length];
+ int byteRead = IOUtils.read(snappyInput, buf);
+ String decompressed = new String(buf, 0, byteRead, "UTF-8");
+ Assert.assertEquals(decompressed, text);
+ } else {
+ System.err.println("WARNING: no hadoop library for this platform. skip hadoop decompression test");
+ }
+ } finally {
+ if (snappyInput != null) {
+ snappyInput.close();
+ }
+ inputFile.delete();
+ snappyFile.delete();
+ outputStream.close();
+ }
+ }
+
+ private void compress(File inputPath, File outputPath) throws Exception
+ {
+ FileInputStream fileInputStream = new FileInputStream(inputPath);
+ FileOutputStream fileOutputStream = new FileOutputStream(outputPath);
+ try {
+ InputStream inputStream = new BufferedInputStream(fileInputStream);
+ OutputStream outputStream = new SnappyHadoopCompatibleOutputStream(fileOutputStream);
+ int readCount;
+ byte[] buffer = new byte[64 * 1024];
+ while ((readCount = inputStream.read(buffer)) > 0) {
+ outputStream.write(buffer, 0, readCount);
+ }
+ inputStream.close();
+ outputStream.close();
+ } finally {
+ fileInputStream.close();
+ fileOutputStream.close();
+ }
+ }
+}
=====================================
src/test/resources/lib/Linux/libhadoop.so
=====================================
Binary files /dev/null and b/src/test/resources/lib/Linux/libhadoop.so differ
=====================================
src/test/resources/lib/Linux/libsnappy.so
=====================================
Binary files /dev/null and b/src/test/resources/lib/Linux/libsnappy.so differ
=====================================
src/test/resources/lib/MacOSX/libhadoop.dylib
=====================================
Binary files /dev/null and b/src/test/resources/lib/MacOSX/libhadoop.dylib differ
=====================================
src/test/resources/lib/MacOSX/libsnappy.dylib
=====================================
Binary files /dev/null and b/src/test/resources/lib/MacOSX/libsnappy.dylib differ
=====================================
src/test/resources/log4j.properties
=====================================
--- /dev/null
+++ b/src/test/resources/log4j.properties
@@ -0,0 +1,8 @@
+log4j.rootLogger = INFO, console
+
+# Set the appender named X to be a File appender
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+
+# Define the layout for X appender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.conversionPattern=%m%n
=====================================
src/test/scala/org/xerial/snappy/SnappyPerformanceTest.scala
=====================================
--- a/src/test/scala/org/xerial/snappy/SnappyPerformanceTest.scala
+++ b/src/test/scala/org/xerial/snappy/SnappyPerformanceTest.scala
@@ -2,14 +2,11 @@ package org.xerial.snappy
import java.io.{ByteArrayOutputStream, ByteArrayInputStream}
-import xerial.core.log.LogLevel
-
import scala.util.Random
-
/**
- *
- */
+ *
+ */
class SnappyPerformanceTest extends SnappySpec {
lazy val data = {
@@ -21,17 +18,16 @@ class SnappyPerformanceTest extends SnappySpec {
a
}
-
"SnappyOutputStream" should {
- "improve output performance" taggedAs("out") in {
+ "improve output performance" taggedAs ("out") in {
val input = data
- time("compression", repeat=100, logLevel = LogLevel.INFO) {
+ time("compression", repeat = 100) {
// 0.037 sec. => 0.026
block("default") {
- val out = new ByteArrayOutputStream()
+ val out = new ByteArrayOutputStream()
val sout = new SnappyOutputStream(out)
sout.write(input)
sout.close()
@@ -43,8 +39,6 @@ class SnappyPerformanceTest extends SnappySpec {
//info(f"compressed size: ${compressed.length}%,d, input: ${data.length}%,d")
}
-
-
}
}
=====================================
src/test/scala/org/xerial/snappy/SnappySpec.scala
=====================================
--- a/src/test/scala/org/xerial/snappy/SnappySpec.scala
+++ b/src/test/scala/org/xerial/snappy/SnappySpec.scala
@@ -1,22 +1,15 @@
package org.xerial.snappy
import org.scalatest._
-import xerial.core.log.Logger
-import xerial.core.util.Timer
+import wvlet.log.LogFormatter.SourceCodeLogFormatter
+import wvlet.log.{LogSupport, Logger}
+import wvlet.log.io.Timer
/**
- *
- */
-trait SnappySpec
- extends WordSpec
- with Matchers
- with GivenWhenThen
- with OptionValues
- with BeforeAndAfter
- with Timer
- with Logger
-{
-
- implicit def toTag(s:String) : Tag = Tag(s)
+ *
+ */
+trait SnappySpec extends WordSpec with Matchers with GivenWhenThen with OptionValues with BeforeAndAfter with Timer with LogSupport {
+ Logger.setDefaultFormatter(SourceCodeLogFormatter)
+ implicit def toTag(s: String): Tag = Tag(s)
}
=====================================
version.sbt
=====================================
--- a/version.sbt
+++ b/version.sbt
@@ -1 +1 @@
-version in ThisBuild := "1.1.4"
+version in ThisBuild := "1.1.7.1"
View it on GitLab: https://salsa.debian.org/java-team/snappy-java/commit/d09f5843676fabf1f4d14091c8881c46ff67d56e
---
View it on GitLab: https://salsa.debian.org/java-team/snappy-java/commit/d09f5843676fabf1f4d14091c8881c46ff67d56e
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-java-commits/attachments/20180502/eae2eaff/attachment.html>
More information about the pkg-java-commits
mailing list