Micro Achat : Ordinateurs, PDA - Toute l\'informatique avec 01Informatique, L\'Ordinateur Individuel, Micro Hebdo, D\351cision Informatique et 01R\351seaux

diff --git a/Makefile b/Makefile index adba9cad..53abb346 100644 --- a/Makefile +++ b/Makefile @@ -27,10 +27,8 @@ BITSHUFFLE_UNPACKED:=$(TARGET)/bitshuffle-extracted.log $(BITSHUFFLE_ARCHIVE): @mkdir -p $(@D) - curl -L -o$@ https://github.com/kiyo-masui/bitshuffle/archive/$(BITSHUFFLE_VERSION).tar.gz $(BITSHUFFLE_UNPACKED): $(BITSHUFFLE_ARCHIVE) - $(TAR) xvfz $< -C $(TARGET) touch $@ $(BITSHUFFLE_SRC): $(BITSHUFFLE_UNPACKED) @@ -67,18 +65,13 @@ endif $(SNAPPY_ARCHIVE): @mkdir -p $(@D) - curl -L -o$@ https://github.com/google/snappy/releases/download/$(SNAPPY_VERSION)/snappy-$(SNAPPY_VERSION).tar.gz $(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE) - $(TAR) xvfz $< -C $(TARGET) touch $@ $(SNAPPY_GIT_UNPACKED): @mkdir -p $(SNAPPY_OUT) - rm -rf $(SNAPPY_SRC_DIR) @mkdir -p $(SNAPPY_SRC_DIR) - git clone $(SNAPPY_GIT_REPO_URL) $(SNAPPY_SRC_DIR) - git --git-dir=$(SNAPPY_SRC_DIR)/.git --work-tree=$(SNAPPY_SRC_DIR) checkout -b local/snappy-$(SNAPPY_VERSION) $(SNAPPY_GIT_REV) touch $@ $(SNAPPY_CMAKE_CACHE): $(SNAPPY_GIT_UNPACKED) diff --git a/Makefile.common b/Makefile.common index 8c060e35..d8a966b2 100755 --- a/Makefile.common +++ b/Makefile.common @@ -99,7 +99,7 @@ ifeq ($(IBM_JDK_7),) else Linux-x86_64_CXXFLAGS := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -m64 -std=c++11 endif -Linux-x86_64_LINKFLAGS := -shared -static-libgcc -static-libstdc++ +Linux-x86_64_LINKFLAGS := -shared -static-libgcc -lstdc++ Linux-x86_64_LIBNAME := libsnappyjava.so Linux-x86_64_SNAPPY_FLAGS := diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..32d2c862 --- /dev/null +++ b/pom.xml @@ -0,0 +1,110 @@ + + + 4.0.0 + org.xerial.snappy + snappy-java + jar + snappy-java: A fast compression/decompression library + 1.1.10.1 + snappy-java + + xerial.org + + https://github.com/xerial/snappy-java + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + leo + Taro L. Saito + leo@xerial.org + Xerial Project + + Architect + Project Manager + Chief Developer + + +9 + + + + GitHub + http://github.com/xerial/snappy-java/issues/list + + 2011 + + scm:git@github.com:xerial/snappy-java.git + scm:git:git@github.com:xerial/snappy-java.git + git@github.com:xerial/snappy-java.git + + + 1.8 + 1.8 + + + + junit + junit + 4.13 + test + + + org.codehaus.plexus + plexus-classworlds + 2.7.0 + test + + + org.xerial.java + xerial-core + 2.1 + test + + + org.wvlet.airframe + airframe-log_2.12 + 23.1.4 + test + + + org.osgi + org.osgi.core + 6.0.0 + provided + + + org.apache.hadoop + hadoop-common + 2.10.2 + test + + + org.xerial.snappy + snappy-java + + + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar + + + + + + + diff --git a/target/bitshuffle-0.3.4/.gitignore b/target/bitshuffle-0.3.4/.gitignore new file mode 100644 index 00000000..d8d6cf49 --- /dev/null +++ b/target/bitshuffle-0.3.4/.gitignore @@ -0,0 +1,77 @@ +## C + +# Object files +*.o +*.ko +*.obj +*.elf + +# Libraries +*.lib +*.a + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + + +## Python +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +__pycache__ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Documentation builds +doc/_build +doc/generated + +## Editor files and backups. +*.swp +*.swo + +# Generated files +bitshuffle/ext.c +bitshuffle/h5.c + diff --git a/target/bitshuffle-0.3.4/.travis.yml b/target/bitshuffle-0.3.4/.travis.yml new file mode 100644 index 00000000..7b5b4994 --- /dev/null +++ b/target/bitshuffle-0.3.4/.travis.yml @@ -0,0 +1,33 @@ +language: python +os: linux +# To test filter plugins, need hdf5 1.8.11+, present in Trusty but not Precise. +dist: trusty +# Required to get Trusty. +#sudo: true +python: + - "2.7" + - "3.4" + - "3.5" + - "3.6" +addons: + apt: + packages: + - libhdf5-serial-dev + - hdf5-tools +install: + - "pip install -U pip virtualenv" + # Ensures the system hdf5 headers/libs will be used whatever its version + - "export HDF5_DIR=/usr/lib" + - "pip install -r requirements.txt" + # Installing the plugin to arbitrary directory to check the install script. + - "python setup.py install --h5plugin --h5plugin-dir ~/hdf5/lib" + # Ensure it's installable and usable in virtualenv + - "virtualenv ~/venv" + - "travis_wait 30 ~/venv/bin/pip -v install --no-binary=h5py ." + - "~/venv/bin/pip -v install nose" +# Can't be somewhere that has a 'bitshuffle' directory as nose will use that +# copy instead of installed package. +script: + - "cd ~" + - "nosetests -v bitshuffle" # Test the system install + - "venv/bin/nosetests -v bitshuffle" # Test the virtualenv install diff --git a/target/bitshuffle-0.3.4/LICENSE b/target/bitshuffle-0.3.4/LICENSE new file mode 100644 index 00000000..1365ed69 --- /dev/null +++ b/target/bitshuffle-0.3.4/LICENSE @@ -0,0 +1,21 @@ +Bitshuffle - Filter for improving compression of typed binary data. + +Copyright (c) 2014 Kiyoshi Masui (kiyo@physics.ubc.ca) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/target/bitshuffle-0.3.4/MANIFEST.in b/target/bitshuffle-0.3.4/MANIFEST.in new file mode 100644 index 00000000..00746c64 --- /dev/null +++ b/target/bitshuffle-0.3.4/MANIFEST.in @@ -0,0 +1,10 @@ +recursive-include src *.h *.c +recursive-include bitshuffle *.pyx +recursive-include lz4 *.h *.c +recursive-include lzf *.h *.c +include setup.cfg.example +include LICENSE +include README.rst +include requirements.txt +exclude setup.cfg + diff --git a/target/bitshuffle-0.3.4/README.rst b/target/bitshuffle-0.3.4/README.rst new file mode 100644 index 00000000..343b4c62 --- /dev/null +++ b/target/bitshuffle-0.3.4/README.rst @@ -0,0 +1,240 @@ +========== +Bitshuffle +========== + +Filter for improving compression of typed binary data. + +Bitshuffle is an algorithm that rearranges typed, binary data for improving +compression, as well as a python/C package that implements this algorithm +within the Numpy framework. + +The library can be used along side HDF5 to compress and decompress datasets and +is integrated through the `dynamically loaded filters`_ framework. Bitshuffle +is HDF5 filter number ``32008``. + +Algorithmically, Bitshuffle is closely related to HDF5's `Shuffle filter`_ +except it operates at the bit level instead of the byte level. Arranging a +typed data array in to a matrix with the elements as the rows and the bits +within the elements as the columns, Bitshuffle "transposes" the matrix, +such that all the least-significant-bits are in a row, etc. This transpose +is performed within blocks of data roughly 8kB long [1]_. + +This does not in itself compress data, only rearranges it for more efficient +compression. To perform the actual compression you will need a compression +library. Bitshuffle has been designed to be well matched Marc Lehmann's +LZF_ as well as LZ4_. Note that because Bitshuffle modifies the data at the bit +level, sophisticated entropy reducing compression libraries such as GZIP and +BZIP are unlikely to achieve significantly better compression than simpler and +faster duplicate-string-elimination algorithms such as LZF and LZ4. Bitshuffle +thus includes routines (and HDF5 filter options) to apply LZ4 compression to +each block after shuffling [2]_. + +The Bitshuffle algorithm relies on neighbouring elements of a dataset being +highly correlated to improve data compression. Any correlations that span at +least 24 elements of the dataset may be exploited to improve compression. + +Bitshuffle was designed with performance in mind. On most machines the +time required for Bitshuffle+LZ4 is insignificant compared to the time required +to read or write the compressed data to disk. Because it is able to exploit the +SSE and AVX instruction sets present on modern Intel and AMD processors, on +these machines compression is only marginally slower than an out-of-cache +memory copy. On modern x86 processors you can expect Bitshuffle to have a +throughput of roughly 1 byte per clock cycle, and on the Haswell generation of +Intel processors (2013) and later, you can expect up to 2 bytes per clock +cycle. In addition, Bitshuffle is parallelized using OpenMP. + +As a bonus, Bitshuffle ships with a dynamically loaded version of +`h5py`'s LZF compression filter, such that the filter can be transparently +used outside of python and in command line utilities such as ``h5dump``. + +.. [1] Chosen to fit comfortably within L1 cache as well as be well matched + window of the LZF compression library. + +.. [2] Over applying bitshuffle to the full dataset then applying LZ4 + compression, this has the tremendous advantage that the block is + already in the L1 cache. + +.. _`dynamically loaded filters`: http://www.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf + +.. _`Shuffle filter`: http://www.hdfgroup.org/HDF5/doc_resource/H5Shuffle_Perf.pdf + +.. _LZF: http://oldhome.schmorp.de/marc/liblzf.html + +.. _LZ4: https://code.google.com/p/lz4/ + + +Applications +------------ + +Bitshuffle might be right for your application if: + +- You need to compress typed binary data. +- Your data is arranged such that adjacent elements over the fastest varying + index of your dataset are similar (highly correlated). +- A special case of the previous point is if you are only exercising a subset + of the bits in your data-type, as is often true of integer data. +- You need both high compression ratios and high performance. + + +Comparing Bitshuffle to other compression algorithms and HDF5 filters: + +- Bitshuffle is less general than many other compression algorithms. + To achieve good compression ratios, consecutive elements of your data must + be highly correlated. +- For the right datasets, Bitshuffle is one of the few compression + algorithms that promises both high throughput and high compression ratios. +- Bitshuffle should have roughly the same throughput as Shuffle, but + may obtain higher compression ratios. +- The MAFISC_ filter actually includes something similar to Bitshuffle as one of + its prefilters, However, MAFICS's emphasis is on obtaining high compression + ratios at all costs, sacrificing throughput. + +.. _MAFISC: http://wr.informatik.uni-hamburg.de/research/projects/icomex/mafisc + + +Installation for Python +----------------------- + +Installation requires python 2.7+ or 3.3+, HDF5 1.8.4 or later, HDF5 for python +(h5py), Numpy and Cython. Bitshuffle must be linked against the same version of +HDF5 as h5py, which in practice means h5py must be built from source_ rather +than pre-built wheels [3]_. To use the dynamically loaded HDF5 filter requires +HDF5 1.8.11 or later. + +To install:: + + python setup.py install [--h5plugin [--h5plugin-dir=spam]] + +To get finer control of installation options, including whether to compile +with OpenMP multi-threading, copy the ``setup.cfg.example`` to ``setup.cfg`` +and edit the values therein. + +If using the dynamically loaded HDF5 filter (which gives you access to the +Bitshuffle and LZF filters outside of python), set the environment variable +``HDF5_PLUGIN_PATH`` to the value of ``--h5plugin-dir`` or use HDF5's default +search location of ``/usr/local/hdf5/lib/plugin``. + +If you get an error about missing source files when building the extensions, +try upgrading setuptools. There is a weird bug where setuptools prior to 0.7 +doesn't work properly with Cython in some cases. + +.. _source: http://docs.h5py.org/en/latest/build.html#source-installation + +.. [3] Typically you will be able to install Bitshuffle, but there will be + errors when creating and reading datasets. + + +Usage from Python +----------------- + +The `bitshuffle` module contains routines for shuffling and unshuffling +Numpy arrays. + +If installed with the dynamically loaded filter plugins, Bitshuffle can be used +in conjunction with HDF5 both inside and outside of python, in the same way as +any other filter; simply by specifying the filter number ``32008``. Otherwise +the filter will be available only within python and only after importing +`bitshuffle.h5`. Reading Bitshuffle encoded datasets will be transparent. +The filter can be added to new datasets either through the `h5py` low level +interface or through the convenience functions provided in +`bitshuffle.h5`. See the docstrings and unit tests for examples. For `h5py` +version 2.5.0 and later Bitshuffle can added to new datasets through the +high level interface, as in the example below. + + +Example h5py +------------ +:: + + import h5py + import numpy + import bitshuffle.h5 + + print(h5py.__version__) # >= '2.5.0' + + f = h5py.File(filename, "w") + + # block_size = 0 let Bitshuffle choose its value + block_size = 0 + + dataset = f.create_dataset( + "data", + (100, 100, 100), + compression=bitshuffle.h5.H5FILTER, + compression_opts=(block_size, bitshuffle.h5.H5_COMPRESS_LZ4), + dtype='float32', + ) + + # create some random data + array = numpy.random.rand(100, 100, 100) + array = array.astype('float32') + + dataset[:] = array + + f.close() + + +Usage from C +------------ + +If you wish to use Bitshuffle in your C program and would prefer not to use the +HDF5 dynamically loaded filter, the C library in the ``src/`` directory is +self-contained and complete. + + +Usage from Java +--------------- + +You can use Bitshuffle even in Java and the routines for shuffling and unshuffling +are ported into `snappy-java`_. To use the routines, you need to add the following +dependency to your pom.xml:: + + + org.xerial.snappy + snappy-java + 1.1.3-M1 + + +First, import org.xerial.snapy.BitShuffle in your Java code:: + + import org.xerial.snappy.BitShuffle; + +Then, you use them like this:: + + int[] data = new int[] {1, 3, 34, 43, 34}; + byte[] shuffledData = BitShuffle.bitShuffle(data); + int[] result = BitShuffle.bitUnShuffleIntArray(shuffledData); + +.. _`snappy-java`: https://github.com/xerial/snappy-java + + +Anaconda +-------- + +The conda package can be build via:: + + conda build conda-recipe + + +For Best Results +---------------- + +Here are a few tips to help you get the most out of Bitshuffle: + +- For multi-dimensional datasets, order your data such that the fastest varying + dimension is the one over which your data is most correlated (have + values that change the least), or fake this using chunks. +- To achieve the highest throughput, use a data type that is 64 *bytes* or + smaller. If you have a very large compound data type, consider adding a + dimension to your datasets instead. +- To make full use of the SSE2 instruction set, use a data type whose size + is a multiple of 2 bytes. For the AVX2 instruction set, use a data type whose + size is a multiple of 4 bytes. + + +Citing Bitshuffle +----------------- + +Bitshuffle was initially described in +http://dx.doi.org/10.1016/j.ascom.2015.07.002, pre-print available at +http://arxiv.org/abs/1503.00638. diff --git a/target/bitshuffle-0.3.4/bitshuffle/__init__.py b/target/bitshuffle-0.3.4/bitshuffle/__init__.py new file mode 100644 index 00000000..8ad676c9 --- /dev/null +++ b/target/bitshuffle-0.3.4/bitshuffle/__init__.py @@ -0,0 +1,20 @@ +""" +Filter for improving compression of typed binary data. + +Functions +========= + + using_SSE2 + using_AVX2 + bitshuffle + bitunshuffle + compress_lz4 + decompress_lz4 + +""" + +from __future__ import absolute_import + + +from bitshuffle.ext import (__version__, bitshuffle, bitunshuffle, using_SSE2, + using_AVX2, compress_lz4, decompress_lz4) diff --git a/target/bitshuffle-0.3.4/bitshuffle/ext.pyx b/target/bitshuffle-0.3.4/bitshuffle/ext.pyx new file mode 100644 index 00000000..cd8a461f --- /dev/null +++ b/target/bitshuffle-0.3.4/bitshuffle/ext.pyx @@ -0,0 +1,410 @@ +""" +Wrappers for public and private bitshuffle routines + +""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import numpy as np + +cimport numpy as np +cimport cython + + +np.import_array() + + +# Repeat each calculation this many times. For timing. +cdef int REPEATC = 1 +#cdef int REPEATC = 32 + +REPEAT = REPEATC + +cdef extern from b"bitshuffle.h": + int bshuf_using_SSE2() + int bshuf_using_AVX2() + int bshuf_bitshuffle(void *A, void *B, int size, int elem_size, + int block_size) + int bshuf_bitunshuffle(void *A, void *B, int size, int elem_size, + int block_size) + int bshuf_compress_lz4_bound(int size, int elem_size, int block_size) + int bshuf_compress_lz4(void *A, void *B, int size, int elem_size, + int block_size) + int bshuf_decompress_lz4(void *A, void *B, int size, int elem_size, + int block_size) + int BSHUF_VERSION_MAJOR + int BSHUF_VERSION_MINOR + int BSHUF_VERSION_POINT + + +__version__ = str("%d.%d.%d").format(BSHUF_VERSION_MAJOR, BSHUF_VERSION_MINOR, + BSHUF_VERSION_POINT) + + +# Prototypes from bitshuffle.c +cdef extern int bshuf_copy(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_byte_elem_scal(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_byte_elem_SSE(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_bit_byte_scal(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_bit_byte_SSE(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_bit_byte_AVX(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_bitrow_eight(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_bit_elem_AVX(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_bit_elem_SSE(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_bit_elem_scal(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_byte_bitrow_SSE(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_byte_bitrow_AVX(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_byte_bitrow_scal(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_shuffle_bit_eightelem_scal(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_shuffle_bit_eightelem_SSE(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_shuffle_bit_eightelem_AVX(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_untrans_bit_elem_SSE(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_untrans_bit_elem_AVX(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_untrans_bit_elem_scal(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_trans_bit_elem(void *A, void *B, int size, int elem_size) +cdef extern int bshuf_untrans_bit_elem(void *A, void *B, int size, int elem_size) + + +ctypedef int (*Cfptr) (void *A, void *B, int size, int elem_size) + + +def using_SSE2(): + """Whether compiled using SSE2 instructions.""" + if bshuf_using_SSE2(): + return True + else: + return False + + +def using_AVX2(): + """Whether compiled using AVX2 instructions.""" + if bshuf_using_AVX2(): + return True + else: + return False + + +def _setup_arr(arr): + shape = tuple(arr.shape) + if not arr.flags['C_CONTIGUOUS']: + msg = "Input array must be C-contiguous." + raise ValueError(msg) + size = arr.size + dtype = arr.dtype + itemsize = dtype.itemsize + out = np.empty(shape, dtype=dtype) + return out, size, itemsize + + +@cython.boundscheck(False) +@cython.wraparound(False) +cdef _wrap_C_fun(Cfptr fun, np.ndarray arr): + """Wrap a C function with standard call signature.""" + + cdef int ii, size, itemsize, count=0 + cdef np.ndarray out + out, size, itemsize = _setup_arr(arr) + + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat + arr_flat = arr.view(np.uint8).ravel() + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat + out_flat = out.view(np.uint8).ravel() + cdef void* arr_ptr = &arr_flat[0] + cdef void* out_ptr = &out_flat[0] + + for ii in range(REPEATC): + count = fun(arr_ptr, out_ptr, size, itemsize) + if count < 0: + msg = "Failed. Error code %d." + excp = RuntimeError(msg % count, count) + raise excp + return out + + +def copy(np.ndarray arr not None): + """Copies the data. + + For testing and profiling purposes. + + """ + return _wrap_C_fun(&bshuf_copy, arr) + + +def trans_byte_elem_scal(np.ndarray arr not None): + """Transpose bytes within words but not bits. + + """ + return _wrap_C_fun(&bshuf_trans_byte_elem_scal, arr) + + +def trans_byte_elem_SSE(np.ndarray arr not None): + """Transpose bytes within array elements. + + """ + return _wrap_C_fun(&bshuf_trans_byte_elem_SSE, arr) + + +def trans_bit_byte_scal(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_bit_byte_scal, arr) + + +def trans_bit_byte_SSE(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_bit_byte_SSE, arr) + + +def trans_bit_byte_AVX(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_bit_byte_AVX, arr) + + +def trans_bitrow_eight(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_bitrow_eight, arr) + + +def trans_bit_elem_AVX(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_bit_elem_AVX, arr) + + +def trans_bit_elem_scal(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_bit_elem_scal, arr) + + +def trans_bit_elem_SSE(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_bit_elem_SSE, arr) + + +def trans_byte_bitrow_SSE(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_byte_bitrow_SSE, arr) + + +def trans_byte_bitrow_AVX(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_byte_bitrow_AVX, arr) + + +def trans_byte_bitrow_scal(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_byte_bitrow_scal, arr) + + +def shuffle_bit_eightelem_scal(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_shuffle_bit_eightelem_scal, arr) + + +def shuffle_bit_eightelem_SSE(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_shuffle_bit_eightelem_SSE, arr) + + +def shuffle_bit_eightelem_AVX(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_shuffle_bit_eightelem_AVX, arr) + + +def untrans_bit_elem_SSE(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_untrans_bit_elem_SSE, arr) + + +def untrans_bit_elem_AVX(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_untrans_bit_elem_AVX, arr) + + +def untrans_bit_elem_scal(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_untrans_bit_elem_scal, arr) + + +def trans_bit_elem(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_trans_bit_elem, arr) + + +def untrans_bit_elem(np.ndarray arr not None): + return _wrap_C_fun(&bshuf_untrans_bit_elem, arr) + + +@cython.boundscheck(False) +@cython.wraparound(False) +def bitshuffle(np.ndarray arr not None, int block_size=0): + """Bitshuffle an array. + + Output array is the same shape and data type as input array but underlying + buffer has been bitshuffled. + + Parameters + ---------- + arr : numpy array + Data to ne processed. + block_size : positive integer + Block size in number of elements. By default, block size is chosen + automatically. + + Returns + ------- + out : numpy array + Array with the same shape as input but underlying data has been + bitshuffled. + + """ + + cdef int ii, size, itemsize, count=0 + cdef np.ndarray out + out, size, itemsize = _setup_arr(arr) + + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat + arr_flat = arr.view(np.uint8).ravel() + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat + out_flat = out.view(np.uint8).ravel() + cdef void* arr_ptr = &arr_flat[0] + cdef void* out_ptr = &out_flat[0] + + for ii in range(REPEATC): + count = bshuf_bitshuffle(arr_ptr, out_ptr, size, itemsize, block_size) + if count < 0: + msg = "Failed. Error code %d." + excp = RuntimeError(msg % count, count) + raise excp + return out + + +@cython.boundscheck(False) +@cython.wraparound(False) +def bitunshuffle(np.ndarray arr not None, int block_size=0): + """Bitshuffle an array. + + Output array is the same shape and data type as input array but underlying + buffer has been un-bitshuffled. + + Parameters + ---------- + arr : numpy array + Data to ne processed. + block_size : positive integer + Block size in number of elements. Must match value used for shuffling. + + Returns + ------- + out : numpy array + Array with the same shape as input but underlying data has been + un-bitshuffled. + + """ + + cdef int ii, size, itemsize, count=0 + cdef np.ndarray out + out, size, itemsize = _setup_arr(arr) + + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat + arr_flat = arr.view(np.uint8).ravel() + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat + out_flat = out.view(np.uint8).ravel() + cdef void* arr_ptr = &arr_flat[0] + cdef void* out_ptr = &out_flat[0] + + for ii in range(REPEATC): + count = bshuf_bitunshuffle(arr_ptr, out_ptr, size, itemsize, block_size) + if count < 0: + msg = "Failed. Error code %d." + excp = RuntimeError(msg % count, count) + raise excp + return out + + +@cython.boundscheck(False) +@cython.wraparound(False) +def compress_lz4(np.ndarray arr not None, int block_size=0): + """Bitshuffle then compress an array using LZ4. + + Parameters + ---------- + arr : numpy array + Data to ne processed. + block_size : positive integer + Block size in number of elements. By default, block size is chosen + automatically. + + Returns + ------- + out : array with np.uint8 data type + Buffer holding compressed data. + + """ + + cdef int ii, size, itemsize, count=0 + shape = (arr.shape[i] for i in range(arr.ndim)) + if not arr.flags['C_CONTIGUOUS']: + msg = "Input array must be C-contiguous." + raise ValueError(msg) + size = arr.size + dtype = arr.dtype + itemsize = dtype.itemsize + + max_out_size = bshuf_compress_lz4_bound(size, itemsize, block_size) + + cdef np.ndarray out + out = np.empty(max_out_size, dtype=np.uint8) + + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat + arr_flat = arr.view(np.uint8).ravel() + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat + out_flat = out.view(np.uint8).ravel() + cdef void* arr_ptr = &arr_flat[0] + cdef void* out_ptr = &out_flat[0] + for ii in range(REPEATC): + count = bshuf_compress_lz4(arr_ptr, out_ptr, size, itemsize, block_size) + if count < 0: + msg = "Failed. Error code %d." + excp = RuntimeError(msg % count, count) + raise excp + return out[:count] + + +@cython.boundscheck(False) +@cython.wraparound(False) +def decompress_lz4(np.ndarray arr not None, shape, dtype, int block_size=0): + """Decompress a buffer using LZ4 then bitunshuffle it yielding an array. + + Parameters + ---------- + arr : numpy array + Input data to be decompressed. + shape : tuple of integers + Shape of the output (decompressed array). Must match the shape of the + original data array before compression. + dtype : numpy dtype + Datatype of the output array. Must match the data type of the original + data array before compression. + block_size : positive integer + Block size in number of elements. Must match value used for + compression. + + Returns + ------- + out : numpy array with shape *shape* and data type *dtype* + Decompressed data. + + """ + + cdef int ii, size, itemsize, count=0 + if not arr.flags['C_CONTIGUOUS']: + msg = "Input array must be C-contiguous." + raise ValueError(msg) + size = np.prod(shape) + itemsize = dtype.itemsize + + cdef np.ndarray out + out = np.empty(tuple(shape), dtype=dtype) + + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat + arr_flat = arr.view(np.uint8).ravel() + cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat + out_flat = out.view(np.uint8).ravel() + cdef void* arr_ptr = &arr_flat[0] + cdef void* out_ptr = &out_flat[0] + for ii in range(REPEATC): + count = bshuf_decompress_lz4(arr_ptr, out_ptr, size, itemsize, + block_size) + if count < 0: + msg = "Failed. Error code %d." + excp = RuntimeError(msg % count, count) + raise excp + if count != arr.size: + msg = "Decompressed different number of bytes than input buffer size." + msg += "Input buffer %d, decompressed %d." % (arr.size, count) + raise RuntimeError(msg, count) + return out + + diff --git a/target/bitshuffle-0.3.4/bitshuffle/h5.pyx b/target/bitshuffle-0.3.4/bitshuffle/h5.pyx new file mode 100644 index 00000000..cd7a0f05 --- /dev/null +++ b/target/bitshuffle-0.3.4/bitshuffle/h5.pyx @@ -0,0 +1,205 @@ +""" +HDF5 support for Bitshuffle. + +To read a dataset that uses the Bitshuffle filter using h5py, simply import +this module (unless you have installed the Bitshuffle dynamically loaded +filter, in which case importing this module is unnecessary). + +To create a new dataset that includes the Bitshuffle filter, use one of the +convenience functions provided. + + +Constants +========= + + H5FILTER : The Bitshuffle HDF5 filter integer identifier. + H5_COMPRESS_LZ4 : Filter option flag for LZ4 compression. + +Functions +========= + + create_dataset + create_bitshuffle_lzf_dataset + create_bitshuffle_compressed_dataset + +Examples +======== + + >>> import numpy as np + >>> import h5py + >>> import bitshuffle.h5 + + >>> shape = (123, 456) + >>> chunks = (10, 456) + >>> dtype = np.float64 + + >>> f = h5py.File("tmp_test.h5") + >>> bitshuffle.h5.create_bitshuffle_compressed_dataset( + f, "some_data", shape, dtype, chunks) + >>> f["some_data"][:] = 42 + +""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import numpy +import h5py +from h5py import h5d, h5s, h5t, h5p, filters + +cimport cython + + +cdef extern from b"bshuf_h5filter.h": + int bshuf_register_h5filter() + int BSHUF_H5FILTER + int BSHUF_H5_COMPRESS_LZ4 + +cdef int LZF_FILTER = 32000 + +H5FILTER = BSHUF_H5FILTER +H5_COMPRESS_LZ4 = BSHUF_H5_COMPRESS_LZ4 + + +def register_h5_filter(): + ret = bshuf_register_h5filter() + if ret < 0: + raise RuntimeError("Failed to register bitshuffle HDF5 filter.", ret) + + +register_h5_filter() + + +def create_dataset(parent, name, shape, dtype, chunks=None, maxshape=None, + fillvalue=None, track_times=None, + filter_pipeline=(), filter_flags=None, filter_opts=None): + """Create a dataset with an arbitrary filter pipeline. + + Return a new low-level dataset identifier. + + Much of this code is copied from h5py, but couldn't reuse much code due to + unstable API. + + """ + + if hasattr(filter_pipeline, "__getitem__"): + filter_pipeline = list(filter_pipeline) + else: + filter_pipeline = [filter_pipeline] + filter_flags = [filter_flags] + filter_opts = [filter_opts] + nfilters = len(filter_pipeline) + if filter_flags is None: + filter_flags = [None] * nfilters + if filter_opts is None: + filter_opts = [None] * nfilters + if not len(filter_flags) == nfilters or not len(filter_opts) == nfilters: + msg = "Supplied incompatible number of filters, flags, and options." + raise ValueError(msg) + + shape = tuple(shape) + + tmp_shape = maxshape if maxshape is not None else shape + # Validate chunk shape + chunks_larger = (numpy.array([ not i>=j + for i,j in zip(tmp_shape,chunks) if i is not None])).any() + if isinstance(chunks, tuple) and chunks_larger: + errmsg = ("Chunk shape must not be greater than data shape in any " + "dimension. {} is not compatible with {}".format(chunks, shape)) + raise ValueError(errmsg) + + if isinstance(dtype, h5py.Datatype): + # Named types are used as-is + tid = dtype.id + dtype = tid.dtype # Following code needs this + else: + # Validate dtype + dtype = numpy.dtype(dtype) + tid = h5t.py_create(dtype, logical=1) + + if shape == (): + if any((chunks, filter_pipeline)): + raise TypeError("Scalar datasets don't support chunk/filter options") + if maxshape and maxshape != (): + raise TypeError("Scalar datasets cannot be extended") + return h5p.create(h5p.DATASET_CREATE) + + def rq_tuple(tpl, name): + """Check if chunks/maxshape match dataset rank""" + if tpl in (None, True): + return + try: + tpl = tuple(tpl) + except TypeError: + raise TypeError('"%s" argument must be None or a sequence object' % name) + if len(tpl) != len(shape): + raise ValueError('"%s" must have same rank as dataset shape' % name) + + rq_tuple(chunks, 'chunks') + rq_tuple(maxshape, 'maxshape') + + if (chunks is True) or (chunks is None and filter_pipeline): + chunks = filters.guess_chunk(shape, maxshape, dtype.itemsize) + + if maxshape is True: + maxshape = (None,)*len(shape) + + dcpl = h5p.create(h5p.DATASET_CREATE) + if chunks is not None: + dcpl.set_chunk(chunks) + dcpl.set_fill_time(h5d.FILL_TIME_ALLOC) # prevent resize glitch + + if fillvalue is not None: + fillvalue = numpy.array(fillvalue) + dcpl.set_fill_value(fillvalue) + + if track_times in (True, False): + dcpl.set_obj_track_times(track_times) + elif track_times is not None: + raise TypeError("track_times must be either True or False") + + for ii in range(nfilters): + this_filter = filter_pipeline[ii] + this_flags = filter_flags[ii] + this_opts = filter_opts[ii] + if this_flags is None: + this_flags = 0 + if this_opts is None: + this_opts = () + dcpl.set_filter(this_filter, this_flags, this_opts) + + if maxshape is not None: + maxshape = tuple(m if m is not None else h5s.UNLIMITED + for m in maxshape) + sid = h5s.create_simple(shape, maxshape) + + dset_id = h5d.create(parent.id, name, tid, sid, dcpl=dcpl) + + return dset_id + + +def create_bitshuffle_lzf_dataset(parent, name, shape, dtype, chunks=None, + maxshape=None, fillvalue=None, + track_times=None): + """Create dataset with a filter pipeline including bitshuffle and LZF""" + + filter_pipeline = [H5FILTER, LZF_FILTER] + dset_id = create_dataset(parent, name, shape, dtype, chunks=chunks, + filter_pipeline=filter_pipeline, maxshape=maxshape, + fillvalue=fillvalue, track_times=track_times) + return dset_id + + +def create_bitshuffle_compressed_dataset(parent, name, shape, dtype, + chunks=None, maxshape=None, + fillvalue=None, track_times=None): + """Create dataset with bitshuffle+internal LZ4 compression.""" + + filter_pipeline = [H5FILTER,] + filter_opts = [(0, H5_COMPRESS_LZ4)] + dset_id = create_dataset(parent, name, shape, dtype, chunks=chunks, + filter_pipeline=filter_pipeline, + filter_opts=filter_opts, maxshape=maxshape, + fillvalue=fillvalue, track_times=track_times) + return dset_id + + diff --git a/target/bitshuffle-0.3.4/bitshuffle/tests/__init__.py b/target/bitshuffle-0.3.4/bitshuffle/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/target/bitshuffle-0.3.4/bitshuffle/tests/data/regression_0.1.3.h5 b/target/bitshuffle-0.3.4/bitshuffle/tests/data/regression_0.1.3.h5 new file mode 100644 index 00000000..ee8373f7 Binary files /dev/null and b/target/bitshuffle-0.3.4/bitshuffle/tests/data/regression_0.1.3.h5 differ diff --git a/target/bitshuffle-0.3.4/bitshuffle/tests/make_regression_tdata.py b/target/bitshuffle-0.3.4/bitshuffle/tests/make_regression_tdata.py new file mode 100644 index 00000000..07045383 --- /dev/null +++ b/target/bitshuffle-0.3.4/bitshuffle/tests/make_regression_tdata.py @@ -0,0 +1,42 @@ +""" +Script to create data used for regression testing. + +""" + +import numpy as np +from numpy import random +import h5py + +import bitshuffle +from bitshuffle import h5 + +BLOCK_SIZE = 64 # Smallish such that datasets have many blocks but are small. +FILTER_PIPELINE = [h5.H5FILTER,] +FILTER_OPTS = [(BLOCK_SIZE, h5.H5_COMPRESS_LZ4)] + +OUT_FILE = "bitshuffle/tests/data/regression_%s.h5" % bitshuffle.__version__ + +DTYPES = ['a1', 'a2', 'a3', 'a4', 'a6', 'a8', 'a10'] + + +f = h5py.File(OUT_FILE, 'w') +g_comp = f.create_group("compressed") +g_orig = f.create_group("origional") + +for dtype in DTYPES: + for rep in ['a', 'b', 'c']: + dset_name = "%s_%s" % (dtype, rep) + dtype = np.dtype(dtype) + n_elem = 3 * BLOCK_SIZE + random.randint(0, BLOCK_SIZE) + shape = (n_elem,) + chunks = shape + data = random.randint(0, 255, n_elem * dtype.itemsize) + data = data.astype(np.uint8).view(dtype) + + g_orig.create_dataset(dset_name, data=data) + + h5.create_dataset(g_comp, dset_name, shape, dtype, chunks=chunks, + filter_pipeline=FILTER_PIPELINE, filter_opts=FILTER_OPTS) + g_comp[dset_name][:] = data + +f.close() diff --git a/target/bitshuffle-0.3.4/bitshuffle/tests/test_ext.py b/target/bitshuffle-0.3.4/bitshuffle/tests/test_ext.py new file mode 100644 index 00000000..11be1ffd --- /dev/null +++ b/target/bitshuffle-0.3.4/bitshuffle/tests/test_ext.py @@ -0,0 +1,588 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import unittest +import time +import timeit + +import numpy as np +from numpy import random + +from bitshuffle import ext + + +# If we are doing timeings by what factor to increase workload. +# Remember to change `ext.REPEATC`. +TIME = 0 +#TIME = 8 # 8kB blocks same as final blocking. +BLOCK = 1024 + + +TEST_DTYPES = [np.uint8, np.uint16, np.int32, np.uint64, np.float32, + np.float64, np.complex128] +TEST_DTYPES += [b'a3', b'a5', b'a6', b'a7', b'a9', b'a11', b'a12', b'a24', + b'a48'] + + +class TestProfile(unittest.TestCase): + + def setUp(self): + n = 1024 # bytes. + if TIME: + n *= TIME + # Almost random bits, but now quite. All bits exercised (to fully test + # transpose) but still slightly compresible. + self.data = random.randint(0, 200, n).astype(np.uint8) + self.fun = ext.copy + self.check = None + self.check_data = None + self.case = "None" + + def tearDown(self): + """Performs all tests and timings.""" + if TIME: + reps = 10 + else: + reps = 1 + delta_ts = [] + try: + for ii in range(reps): + t0 = time.time() + out = self.fun(self.data) + delta_ts.append(time.time() - t0) + except RuntimeError as err: + if (len(err.args) > 1 and (err.args[1] == -11) + and not ext.using_SSE2()): + return + if (len(err.args) > 1 and (err.args[1] == -12) + and not ext.using_AVX2()): + return + else: + raise + delta_t = min(delta_ts) + size_i = self.data.size * self.data.dtype.itemsize + size_o = out.size * out.dtype.itemsize + size = max([size_i, size_o]) + speed = (ext.REPEAT * size / delta_t / 1024**3) # GB/s + if TIME: + print("%-20s: %5.2f s/GB, %5.2f GB/s" % (self.case, 1./speed, speed)) + if not self.check is None: + ans = self.check(self.data).view(np.uint8) + self.assertTrue(np.all(ans == out.view(np.uint8))) + if not self.check_data is None: + ans = self.check_data.view(np.uint8) + self.assertTrue(np.all(ans == out.view(np.uint8))) + + def test_00_copy(self): + self.case = "copy" + self.fun = ext.copy + self.check = lambda x: x + + def test_01a_trans_byte_elem_scal_16(self): + self.case = "byte T elem scal 16" + self.data = self.data.view(np.int16) + self.fun = ext.trans_byte_elem_scal + self.check = trans_byte_elem + + def test_01b_trans_byte_elem_scal_32(self): + self.case = "byte T elem scal 32" + self.data = self.data.view(np.int32) + self.fun = ext.trans_byte_elem_scal + self.check = trans_byte_elem + + def test_01c_trans_byte_elem_scal_64(self): + self.case = "byte T elem scal 64" + self.data = self.data.view(np.int64) + self.fun = ext.trans_byte_elem_scal + self.check = trans_byte_elem + + def test_01d_trans_byte_elem_16(self): + self.case = "byte T elem SSE 16" + self.data = self.data.view(np.int16) + self.fun = ext.trans_byte_elem_SSE + self.check = trans_byte_elem + + def test_01e_trans_byte_elem_32(self): + self.case = "byte T elem SSE 32" + self.data = self.data.view(np.float32) + self.fun = ext.trans_byte_elem_SSE + self.check = trans_byte_elem + + def test_01f_trans_byte_elem_64(self): + self.case = "byte T elem SSE 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_byte_elem_SSE + self.check = trans_byte_elem + + def test_01g_trans_byte_elem_128(self): + self.case = "byte T elem SSE 128" + self.data = self.data.view(np.complex128) + self.fun = ext.trans_byte_elem_SSE + self.check = trans_byte_elem + + def test_01h_trans_byte_elem_96(self): + self.case = "byte T elem SSE 96" + n = self.data.size // 128 * 96 + dt = np.dtype([(str('a'), np.int32), (str('b'), np.int32), + (str('c'), np.int32)]) + self.data = self.data[:n].view(dt) + self.fun = ext.trans_byte_elem_SSE + self.check = trans_byte_elem + + def test_01i_trans_byte_elem_80(self): + self.case = "byte T elem SSE 80" + n = self.data.size // 128 * 80 + dt = np.dtype([(str('a'), np.int16), (str('b'), np.int16), + (str('c'), np.int16), (str('d'), np.int16), + (str('e'), np.int16)]) + self.data = self.data[:n].view(dt) + self.fun = ext.trans_byte_elem_SSE + self.check = trans_byte_elem + + def test_03a_trans_bit_byte(self): + self.case = "bit T byte scal 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_byte_scal + self.check = trans_bit_byte + + def test_03d_trans_bit_byte_SSE(self): + self.case = "bit T byte SSE 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_byte_SSE + self.check = trans_bit_byte + + def test_03f_trans_bit_byte_AVX(self): + self.case = "bit T byte AVX 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_byte_AVX + self.check = trans_bit_byte + + def test_03g_trans_bit_byte_AVX_32(self): + self.case = "bit T byte AVX 32" + self.data = self.data.view(np.float32) + self.fun = ext.trans_bit_byte_AVX + self.check = trans_bit_byte + + def test_04a_trans_bit_elem_AVX(self): + self.case = "bit T elem AVX 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_elem_AVX + self.check = trans_bit_elem + + def test_04b_trans_bit_elem_AVX_128(self): + self.case = "bit T elem AVX 128" + self.data = self.data.view(np.complex128) + self.fun = ext.trans_bit_elem_AVX + self.check = trans_bit_elem + + def test_04c_trans_bit_elem_AVX_32(self): + self.case = "bit T elem AVX 32" + self.data = self.data.view(np.float32) + self.fun = ext.trans_bit_elem_AVX + self.check = trans_bit_elem + + def test_04d_trans_bit_elem_AVX_16(self): + self.case = "bit T elem AVX 16" + self.data = self.data.view(np.int16) + self.fun = ext.trans_bit_elem_AVX + self.check = trans_bit_elem + + def test_04e_trans_bit_elem_64(self): + self.case = "bit T elem scal 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_elem_scal + self.check = trans_bit_elem + + def test_04f_trans_bit_elem_SSE_32(self): + self.case = "bit T elem SSE 32" + self.data = self.data.view(np.float32) + self.fun = ext.trans_bit_elem_SSE + self.check = trans_bit_elem + + def test_04g_trans_bit_elem_SSE_64(self): + self.case = "bit T elem SSE 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_elem_SSE + self.check = trans_bit_elem + + def test_06a_untrans_bit_elem_16(self): + self.case = "bit U elem SSE 16" + pre_trans = self.data.view(np.int16) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_SSE + self.check_data = pre_trans + + def test_06b_untrans_bit_elem_128(self): + self.case = "bit U elem SSE 128" + pre_trans = self.data.view(np.complex128) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_SSE + self.check_data = pre_trans + + def test_06c_untrans_bit_elem_32(self): + self.case = "bit U elem SSE 32" + pre_trans = self.data.view(np.float32) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_SSE + self.check_data = pre_trans + + def test_06d_untrans_bit_elem_32(self): + self.case = "bit U elem AVX 32" + pre_trans = self.data.view(np.float32) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_AVX + self.check_data = pre_trans + + def test_06e_untrans_bit_elem_64(self): + self.case = "bit U elem SSE 64" + pre_trans = self.data.view(np.float64) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_SSE + self.check_data = pre_trans + + def test_06f_untrans_bit_elem_64(self): + self.case = "bit U elem AVX 64" + pre_trans = self.data.view(np.float64) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_AVX + self.check_data = pre_trans + + def test_06g_untrans_bit_elem_64(self): + self.case = "bit U elem scal 64" + pre_trans = self.data.view(np.float64) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_scal + self.check_data = pre_trans + + def test_07a_trans_byte_bitrow_64(self): + self.case = "byte T row scal 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_byte_bitrow_scal + + def test_07b_trans_byte_bitrow_SSE_64(self): + self.case = "byte T row SSE 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_byte_bitrow_SSE + self.check = ext.trans_byte_bitrow_scal + + def test_07c_trans_byte_bitrow_AVX_64(self): + self.case = "byte T row AVX 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_byte_bitrow_AVX + self.check = ext.trans_byte_bitrow_scal + + def test_08a_shuffle_bit_eight_scal_64(self): + self.case = "bit S eight scal 64" + self.data = self.data.view(np.float64) + self.fun = ext.shuffle_bit_eightelem_scal + + def test_08b_shuffle_bit_eight_SSE_64(self): + self.case = "bit S eight SSE 64" + self.data = self.data.view(np.float64) + self.fun = ext.shuffle_bit_eightelem_SSE + self.check = ext.shuffle_bit_eightelem_scal + + def test_08c_shuffle_bit_eight_AVX_32(self): + self.case = "bit S eight AVX 32" + self.data = self.data.view(np.float32) + self.fun = ext.shuffle_bit_eightelem_AVX + self.check = ext.shuffle_bit_eightelem_scal + + def test_08d_shuffle_bit_eight_AVX_64(self): + self.case = "bit S eight AVX 64" + self.data = self.data.view(np.float64) + self.fun = ext.shuffle_bit_eightelem_AVX + self.check = ext.shuffle_bit_eightelem_scal + + def test_08e_shuffle_bit_eight_AVX_16(self): + self.case = "bit S eight AVX 16" + self.data = self.data.view(np.int16) + self.fun = ext.shuffle_bit_eightelem_AVX + self.check = ext.shuffle_bit_eightelem_scal + + def test_08f_shuffle_bit_eight_AVX_128(self): + self.case = "bit S eight AVX 128" + self.data = self.data.view(np.complex128) + self.fun = ext.shuffle_bit_eightelem_AVX + self.check = ext.shuffle_bit_eightelem_scal + + def test_09a_trans_bit_elem_scal_64(self): + self.case = "bit T elem scal 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_elem_scal + self.check = trans_bit_elem + + def test_09b_trans_bit_elem_SSE_64(self): + self.case = "bit T elem SSE 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_elem_SSE + self.check = trans_bit_elem + + def test_09c_trans_bit_elem_AVX_64(self): + self.case = "bit T elem AVX 64" + self.data = self.data.view(np.float64) + self.fun = ext.trans_bit_elem_AVX + self.check = trans_bit_elem + + def test_09d_untrans_bit_elem_scal_64(self): + self.case = "bit U elem scal 64" + pre_trans = self.data.view(np.float64) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_scal + self.check_data = pre_trans + + def test_09e_untrans_bit_elem_SSE_64(self): + self.case = "bit U elem SSE 64" + pre_trans = self.data.view(np.float64) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_SSE + self.check_data = pre_trans + + def test_09f_untrans_bit_elem_AVX_64(self): + self.case = "bit U elem AVX 64" + pre_trans = self.data.view(np.float64) + self.data = trans_bit_elem(pre_trans) + self.fun = ext.untrans_bit_elem_AVX + self.check_data = pre_trans + + def test_10a_bitshuffle_64(self): + self.case = "bitshuffle 64" + self.data = self.data.view(np.float64) + self.fun = lambda x: ext.bitshuffle(x, BLOCK) + + def test_10b_bitunshuffle_64(self): + self.case = "bitunshuffle 64" + pre_trans = self.data.view(np.float64) + self.data = ext.bitshuffle(pre_trans, BLOCK) + self.fun = lambda x: ext.bitunshuffle(x, BLOCK) + self.check_data = pre_trans + + def test_10c_compress_64(self): + self.case = "compress 64" + self.data = self.data.view(np.float64) + self.fun = lambda x:ext.compress_lz4(x, BLOCK) + + def test_10d_decompress_64(self): + self.case = "decompress 64" + pre_trans = self.data.view(np.float64) + self.data = ext.compress_lz4(pre_trans, BLOCK) + self.fun = lambda x: ext.decompress_lz4(x, pre_trans.shape, + pre_trans.dtype, BLOCK) + self.check_data = pre_trans + +""" +Commented out to prevent nose from finding them. +class TestDevCases(unittest.TestCase): + + def deactivated_test_trans_byte_bitrow_AVX(self): + d = np.arange(256, dtype=np.uint32) + #d = ext.trans_bit_elem(d) + t = ext.trans_byte_bitrow_AVX(d).view(np.uint8) + t1 = ext.trans_byte_bitrow_SSE(d).view(np.uint8) + t.shape = (32, 32) + t1.shape = (32, 32) + #print t[:20,:18] + self.assertTrue(np.all(t == t1)) + + def deactivated_test_untrans_bit_elem(self): + d = np.arange(32, dtype=np.uint16) + #d = random.randint(0, 2**7, 256).astype(np.uint16) + d1 = ext.trans_bit_elem(d) + #print d + t = ext.untrans_bit_elem_AVX(d1) + #t1 = ext.untrans_bit_byte_scal(d1) + #print np.reshape(d1.view(np.uint8), (16, 4)) + #print np.reshape(t1.view(np.uint8), (2, 32)) + #print np.reshape(t2.view(np.uint8), (32, 2)) + #print np.reshape(t.view(np.uint8), (32, 2)) + + def deactivated_test_trans_bit_byte(self): + d = np.arange(16, dtype=np.uint16) + t = ext.trans_bit_byte_scal(d) + #print t + t1 = trans_bit_byte(d) + #print t1 + self.assertTrue(np.all(t == t1)) + + def deactivated_test_trans_byte_bitrow_SSE(self): + d = np.arange(256, dtype = np.uint8) + t = ext.trans_byte_bitrow_scal(d) + #print np.reshape(t, (32, 8)) + t1 = ext.trans_byte_bitrow_SSE(d) + #print np.reshape(t1, (32, 8)) + self.assertTrue(np.all(t == t1)) + + def deactivated_test_trans_byte_elem_SSE(self): + d = np.empty(16, dtype=([('a', 'u4'), ('b', 'u4'), ('c', 'u4')])) + d['a'] = np.arange(16) * 1 + d['b'] = np.arange(16) * 2 + d['c'] = np.arange(16) * 3 + #print d.dtype.itemsize + #print np.reshape(d.view(np.uint8), (16, 12)) + t1 = ext.trans_byte_elem_SSE(d) + #print np.reshape(t1.view(np.uint8), (12, 16)) + t0 = trans_byte_elem(d) + #print np.reshape(t0.view(np.uint8), (12, 16)) + self.assertTrue(np.all(t0.view(np.uint8) == t1.view(np.uint8))) + + def deactivated_test_bitshuffle(self): + d = np.arange(128, dtype=np.uint16) + t1 = ext.bitshuffle(d) + #print t1 + t2 = ext.bitunshuffle(t1) + #print t2 + self.assertTrue(np.all(t2.view(np.uint8) == d.view(np.uint8))) +""" + + +class TestOddLengths(unittest.TestCase): + + def setUp(self): + self.reps = 10 + self.nmax = 128 * 8 + #self.nmax = 4 * 8 # XXX + self.fun = ext.copy + self.check = lambda x: x + + def test_trans_bit_elem_SSE(self): + self.fun = ext.trans_bit_elem_SSE + self.check = trans_bit_elem + + def test_untrans_bit_elem_SSE(self): + self.fun = lambda x: ext.untrans_bit_elem_SSE(ext.trans_bit_elem(x)) + self.check = lambda x: x + + def test_trans_bit_elem_AVX(self): + self.fun = ext.trans_bit_elem_AVX + self.check = trans_bit_elem + + def test_untrans_bit_elem_AVX(self): + self.fun = lambda x: ext.untrans_bit_elem_SSE(ext.trans_bit_elem(x)) + self.check = lambda x: x + + def test_trans_bit_elem_scal(self): + self.fun = ext.trans_bit_elem_scal + self.check = trans_bit_elem + + def test_untrans_bit_elem_scal(self): + self.fun = lambda x: ext.untrans_bit_elem_scal(ext.trans_bit_elem(x)) + self.check = lambda x: x + + def test_trans_byte_elem_SSE(self): + self.fun = ext.trans_byte_elem_SSE + self.check = trans_byte_elem + + def tearDown(self): + try: + for dtype in TEST_DTYPES: + itemsize = np.dtype(dtype).itemsize + nbyte_max = self.nmax * itemsize + dbuf = random.randint(0, 255, nbyte_max).astype(np.uint8) + dbuf = dbuf.view(dtype) + for ii in range(self.reps): + n = random.randint(0, self.nmax // 8, 1)[0] * 8 + data = dbuf[:n] + out = self.fun(data).view(np.uint8) + ans = self.check(data).view(np.uint8) + self.assertTrue(np.all(out == ans)) + except RuntimeError as err: + if (len(err.args) > 1 and (err.args[1] == -11) + and not ext.using_SSE2()): + return + if (len(err.args) > 1 and (err.args[1] == -12) + and not ext.using_AVX2()): + return + else: + raise + + +class TestBitShuffleCircle(unittest.TestCase): + """Ensure that final filter is circularly consistant for any data type and + any length buffer.""" + + def test_circle(self): + nmax = 100000 + reps = 20 + for dtype in TEST_DTYPES: + itemsize = np.dtype(dtype).itemsize + nbyte_max = nmax * itemsize + dbuf = random.randint(0, 255, nbyte_max).astype(np.uint8) + dbuf = dbuf.view(dtype) + for ii in range(reps): + n = random.randint(0, nmax, 1)[0] + data = dbuf[:n] + shuff = ext.bitshuffle(data) + out = ext.bitunshuffle(shuff) + self.assertTrue(out.dtype is data.dtype) + self.assertTrue(np.all(data.view(np.uint8) + == out.view(np.uint8))) + + def test_circle_with_compression(self): + nmax = 100000 + reps = 20 + for dtype in TEST_DTYPES: + itemsize = np.dtype(dtype).itemsize + nbyte_max = nmax * itemsize + dbuf = random.randint(0, 255, nbyte_max).astype(np.uint8) + dbuf = dbuf.view(dtype) + for ii in range(reps): + n = random.randint(0, nmax, 1)[0] + data = dbuf[:n] + shuff = ext.compress_lz4(data) + out = ext.decompress_lz4(shuff, data.shape, data.dtype) + self.assertTrue(out.dtype is data.dtype) + self.assertTrue(np.all(data.view(np.uint8) + == out.view(np.uint8))) + + +# Python implementations for checking results. + +def trans_byte_elem(arr): + dtype = arr.dtype + itemsize = dtype.itemsize + in_buf = arr.flat[:].view(np.uint8) + nelem = in_buf.size // itemsize + in_buf.shape = (nelem, itemsize) + + out_buf = np.empty((itemsize, nelem), dtype=np.uint8) + for ii in range(nelem): + for jj in range(itemsize): + out_buf[jj,ii] = in_buf[ii,jj] + return out_buf.flat[:].view(dtype) + + +def trans_bit_byte(arr): + n = arr.size + dtype = arr.dtype + itemsize = dtype.itemsize + bits = np.unpackbits(arr.view(np.uint8)) + bits.shape = (n * itemsize, 8) + # We have to reverse the order of the bits both for unpacking and packing, + # since we want to call the least significant bit the first bit. + bits = bits[:,::-1] + bits_shuff = (bits.T).copy() + bits_shuff.shape = (n * itemsize, 8) + bits_shuff = bits_shuff[:,::-1] + arr_bt = np.packbits(bits_shuff.flat[:]) + return arr_bt.view(dtype) + + +def trans_bit_elem(arr): + n = arr.size + dtype = arr.dtype + itemsize = dtype.itemsize + bits = np.unpackbits(arr.view(np.uint8)) + bits.shape = (n * itemsize, 8) + # We have to reverse the order of the bits both for unpacking and packing, + # since we want to call the least significant bit the first bit. + bits = bits[:,::-1].copy() + bits.shape = (n, itemsize * 8) + bits_shuff = (bits.T).copy() + bits_shuff.shape = (n * itemsize, 8) + bits_shuff = bits_shuff[:,::-1] + arr_bt = np.packbits(bits_shuff.flat[:]) + return arr_bt.view(dtype) + + + +if __name__ == "__main__": + unittest.main() diff --git a/target/bitshuffle-0.3.4/bitshuffle/tests/test_h5filter.py b/target/bitshuffle-0.3.4/bitshuffle/tests/test_h5filter.py new file mode 100644 index 00000000..6739b998 --- /dev/null +++ b/target/bitshuffle-0.3.4/bitshuffle/tests/test_h5filter.py @@ -0,0 +1,91 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import unittest +import os +import glob + +import numpy as np +import h5py +from h5py import h5f, h5d, h5z, h5t, h5s, filters +from subprocess import Popen, PIPE, STDOUT + +from bitshuffle import h5 + + +os.environ["HDF5_PLUGIN_PATH"] = "" + + +class TestFilter(unittest.TestCase): + + def test_filter(self): + shape = (32 * 1024 + 783,) + chunks = (4 * 1024 + 23,) + dtype = np.int64 + data = np.arange(shape[0]) + fname = "tmp_test_filters.h5" + f = h5py.File(fname) + h5.create_dataset(f, b"range", shape, dtype, chunks, + filter_pipeline=(32008, 32000), + filter_flags=(h5z.FLAG_MANDATORY, h5z.FLAG_MANDATORY), + filter_opts=None) + f["range"][:] = data + + f.close() + + f = h5py.File(fname, 'r') + d = f['range'][:] + self.assertTrue(np.all(d == data)) + f.close() + + def test_with_block_size(self): + shape = (128 * 1024 + 783,) + chunks = (4 * 1024 + 23,) + dtype = np.int64 + data = np.arange(shape[0]) + fname = "tmp_test_filters.h5" + f = h5py.File(fname) + h5.create_dataset(f, b"range", shape, dtype, chunks, + filter_pipeline=(32008, 32000), + filter_flags=(h5z.FLAG_MANDATORY, h5z.FLAG_MANDATORY), + filter_opts=((680,), ()), + ) + f["range"][:] = data + + f.close() + #os.system('h5dump -H -p tmp_test_filters.h5') + + f = h5py.File(fname, 'r') + d = f['range'][:] + self.assertTrue(np.all(d == data)) + f.close() + + def test_with_compression(self): + shape = (128 * 1024 + 783,) + chunks = (4 * 1024 + 23,) + dtype = np.int64 + data = np.arange(shape[0]) + fname = "tmp_test_filters.h5" + f = h5py.File(fname) + h5.create_dataset(f, b"range", shape, dtype, chunks, + filter_pipeline=(32008,), + filter_flags=(h5z.FLAG_MANDATORY,), + filter_opts=((0, h5.H5_COMPRESS_LZ4),), + ) + f["range"][:] = data + + f.close() + #os.system('h5dump -H -p tmp_test_filters.h5') + + f = h5py.File(fname, 'r') + d = f['range'][:] + self.assertTrue(np.all(d == data)) + f.close() + + def tearDown(self): + files = glob.glob("tmp_test_*") + for f in files: + os.remove(f) + + +if __name__ == "__main__": + unittest.main() diff --git a/target/bitshuffle-0.3.4/bitshuffle/tests/test_h5plugin.py b/target/bitshuffle-0.3.4/bitshuffle/tests/test_h5plugin.py new file mode 100644 index 00000000..220d55da --- /dev/null +++ b/target/bitshuffle-0.3.4/bitshuffle/tests/test_h5plugin.py @@ -0,0 +1,83 @@ +from __future__ import absolute_import, division, print_function, unicode_literals +import unittest +import os, os.path +import glob + +import numpy as np +import h5py +from h5py import h5f, h5d, h5z, h5t, h5s, filters +from subprocess import Popen, PIPE, STDOUT + +import bitshuffle + + +plugin_dir = os.path.join(os.path.dirname(bitshuffle.__file__), + 'plugin') +os.environ["HDF5_PLUGIN_PATH"] = plugin_dir + + +H5VERSION = h5py.h5.get_libversion() +if (H5VERSION[0] < 1 or (H5VERSION[0] == 1 + and (H5VERSION[1] < 8 or (H5VERSION[1] == 8 and H5VERSION[2] < 11)))): + H51811P = False +else: + H51811P = True + + +class TestFilterPlugins(unittest.TestCase): + + def test_plugins(self): + if not H51811P: + return + shape = (32 * 1024,) + chunks = (4 * 1024,) + dtype = np.int64 + data = np.arange(shape[0]) + fname = "tmp_test_filters.h5" + f = h5py.File(fname) + tid = h5t.py_create(dtype, logical=1) + sid = h5s.create_simple(shape, shape) + # Different API's for different h5py versions. + try: + dcpl = filters.generate_dcpl(shape, dtype, chunks, None, None, + None, None, None, None) + except TypeError: + dcpl = filters.generate_dcpl(shape, dtype, chunks, None, None, + None, None, None) + dcpl.set_filter(32008, h5z.FLAG_MANDATORY) + dcpl.set_filter(32000, h5z.FLAG_MANDATORY) + dset_id = h5d.create(f.id, b"range", tid, sid, dcpl=dcpl) + dset_id.write(h5s.ALL, h5s.ALL, data) + f.close() + + # Make sure the filters are working outside of h5py by calling h5dump + h5dump = Popen(['h5dump', fname], + stdout=PIPE, stderr=STDOUT) + stdout, nothing = h5dump.communicate() + err = h5dump.returncode + self.assertEqual(err, 0) + + + f = h5py.File(fname, 'r') + d = f['range'][:] + self.assertTrue(np.all(d == data)) + f.close() + + + #def test_h5py_hl(self): + # if not H51811P: + # return + # # Does not appear to be supported by h5py. + # fname = "tmp_test_h5py_hl.h5" + # f = h5py.File(fname) + # f.create_dataset("range", np.arange(1024, dtype=np.int64), + # compression=32008) + + def tearDown(self): + files = glob.glob("tmp_test_*") + for f in files: + os.remove(f) + + +if __name__ == "__main__": + unittest.main() diff --git a/target/bitshuffle-0.3.4/bitshuffle/tests/test_regression.py b/target/bitshuffle-0.3.4/bitshuffle/tests/test_regression.py new file mode 100644 index 00000000..2862cace --- /dev/null +++ b/target/bitshuffle-0.3.4/bitshuffle/tests/test_regression.py @@ -0,0 +1,40 @@ +""" +Test that data encoded with earlier versions can still be decoded correctly. + +""" + +from __future__ import absolute_import, division, print_function + +import unittest +from os import path + +import numpy as np +import h5py + +import bitshuffle +from bitshuffle import h5 + + +TEST_DATA_DIR = path.dirname(bitshuffle.__file__) + "/tests/data" + +OUT_FILE_TEMPLATE = TEST_DATA_DIR + "/regression_%s.h5" + +VERSIONS = ["0.1.3",] + + +class TestAll(unittest.TestCase): + + def test_regression(self): + for version in VERSIONS: + file_name = OUT_FILE_TEMPLATE % version + f = h5py.File(file_name) + g_orig = f["origional"] + g_comp = f["compressed"] + + for dset_name in g_comp.keys(): + self.assertTrue(np.all(g_comp[dset_name][:] + == g_orig[dset_name][:])) + + +if __name__ == "__main__": + unittest.main() diff --git a/target/bitshuffle-0.3.4/conda-recipe/bld.bat b/target/bitshuffle-0.3.4/conda-recipe/bld.bat new file mode 100644 index 00000000..ccbb10f9 --- /dev/null +++ b/target/bitshuffle-0.3.4/conda-recipe/bld.bat @@ -0,0 +1,3 @@ +SET CONDA_HOME=%PREFIX% +"%PYTHON%" setup.py install +if errorlevel 1 exit 1 diff --git a/target/bitshuffle-0.3.4/conda-recipe/build.sh b/target/bitshuffle-0.3.4/conda-recipe/build.sh new file mode 100644 index 00000000..34c3a689 --- /dev/null +++ b/target/bitshuffle-0.3.4/conda-recipe/build.sh @@ -0,0 +1,2 @@ +export CONDA_HOME=$PREFIX +$PYTHON setup.py install # Python command to install the script diff --git a/target/bitshuffle-0.3.4/conda-recipe/meta.yaml b/target/bitshuffle-0.3.4/conda-recipe/meta.yaml new file mode 100644 index 00000000..ac227e2b --- /dev/null +++ b/target/bitshuffle-0.3.4/conda-recipe/meta.yaml @@ -0,0 +1,27 @@ +package: + name: bitshuffle + version: 0.2.1 +source: + # git_url: https://github.com/kiyo-masui/bitshuffle.git + # git_rev: 0.2.1 + path: .. + patches: + - setup.py.patch + +requirements: + build: + - python + - setuptools + - cython + - numpy + - h5py + - hdf5 + run: + - python + - numpy + - h5py + - cython + +about: + home: https://github.com/kiyo-masui/bitshuffle/blob/master/setup.py + summary: "bitshuffle library." diff --git a/target/bitshuffle-0.3.4/conda-recipe/setup.py.patch b/target/bitshuffle-0.3.4/conda-recipe/setup.py.patch new file mode 100644 index 00000000..437a5ffa --- /dev/null +++ b/target/bitshuffle-0.3.4/conda-recipe/setup.py.patch @@ -0,0 +1,13 @@ +--- setup.py 2016-01-19 16:56:12.954563000 +0100 ++++ xxx.py 2016-01-19 16:56:00.817087000 +0100 +@@ -40,8 +40,8 @@ + + # Copied from h5py. + # TODO, figure out what the canonacal way to do this should be. +-INCLUDE_DIRS = [] +-LIBRARY_DIRS = [] ++INCLUDE_DIRS = [os.environ['CONDA_HOME'] + '/include'] ++LIBRARY_DIRS = [os.environ['CONDA_HOME'] + '/lib'] + if sys.platform == 'darwin': + # putting here both macports and homebrew paths will generate + # "ld: warning: dir not found" at the linking phase diff --git a/target/bitshuffle-0.3.4/lz4/LICENSE b/target/bitshuffle-0.3.4/lz4/LICENSE new file mode 100644 index 00000000..b566df30 --- /dev/null +++ b/target/bitshuffle-0.3.4/lz4/LICENSE @@ -0,0 +1,24 @@ +LZ4 Library +Copyright (c) 2011-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/target/bitshuffle-0.3.4/lz4/README.md b/target/bitshuffle-0.3.4/lz4/README.md new file mode 100644 index 00000000..f6ebf5e1 --- /dev/null +++ b/target/bitshuffle-0.3.4/lz4/README.md @@ -0,0 +1,21 @@ +LZ4 - Library Files +================================ + +The __lib__ directory contains several files, but you don't necessarily need them all. + +To integrate fast LZ4 compression/decompression into your program, you basically just need "**lz4.c**" and "**lz4.h**". + +For more compression at the cost of compression speed (while preserving decompression speed), use **lz4hc** on top of regular lz4. `lz4hc` only provides compression functions. It also needs `lz4` to compile properly. + +If you want to produce files or data streams compatible with `lz4` command line utility, use **lz4frame**. This library encapsulates lz4-compressed blocks into the [official interoperable frame format]. In order to work properly, lz4frame needs lz4 and lz4hc, and also **xxhash**, which provides error detection algorithm. +(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace. This is what `liblz4` does, to avoid symbol duplication in case a user program would link to several libraries containing xxhash symbols.) + +A more complex "lz4frame_static.h" is also provided, although its usage is not recommended. It contains definitions which are not guaranteed to remain stable within future versions. Use for static linking ***only***. + +The other files are not source code. There are : + + - LICENSE : contains the BSD license text + - Makefile : script to compile or install lz4 library (static or dynamic) + - liblz4.pc.in : for pkg-config (make install) + +[official interoperable frame format]: ../lz4_Frame_format.md diff --git a/target/bitshuffle-0.3.4/lz4/lz4.c b/target/bitshuffle-0.3.4/lz4/lz4.c new file mode 100644 index 00000000..08cf6b5c --- /dev/null +++ b/target/bitshuffle-0.3.4/lz4/lz4.c @@ -0,0 +1,1516 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + + +/************************************** +* Tuning parameters +**************************************/ +/* + * HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#define HEAPMODE 0 + +/* + * ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define ACCELERATION_DEFAULT 1 + + +/************************************** +* CPU Feature Detection +**************************************/ +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +/************************************** +* Includes +**************************************/ +#include "lz4.h" + + +/************************************** +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# if defined(__GNUC__) || defined(__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif /* _MSC_VER */ + +/* LZ4_GCC_VERSION is defined into lz4.h */ +#if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + +/************************************** +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include /* memset, memcpy */ +#define MEM_INIT memset + + +/************************************** +* Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/************************************** +* Reading and writing into memory +**************************************/ +#define STEPSIZE sizeof(size_t) + +static unsigned LZ4_64bits(void) { return sizeof(void*)==8; } + +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val16; + memcpy(&val16, memPtr, 2); + return val16; +} + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) + { + return LZ4_read16(memPtr); + } + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) + { + memcpy(memPtr, &value, 2); + } + else + { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; +} + +static U64 LZ4_read64(const void* memPtr) +{ + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; +} + +static size_t LZ4_read_ARCH(const void* p) +{ + if (LZ4_64bits()) + return (size_t)LZ4_read64(p); + else + return (size_t)LZ4_read32(p); +} + + +static void LZ4_copy4(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 4); } + +static void LZ4_copy8(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 8); } + +/* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ +static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* e = (BYTE*)dstEnd; + do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } + else /* Big Endian CPU */ + { + if (LZ4_64bits()) + { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + +static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn compression run slower on incompressible data */ + + +/************************************** +* Local Structures and types +**************************************/ +typedef struct { + U32 hashTable[HASH_SIZE_U32]; + U32 currentOffset; + U32 initCheck; + const BYTE* dictionary; + BYTE* bufferStart; /* obsolete, used for slideInputBuffer */ + U32 dictSize; +} LZ4_stream_t_internal; + +typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; +typedef enum { byPtr, byU32, byU16 } tableType_t; + +typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { full = 0, partial = 1 } earlyEnd_directive; + + +/************************************** +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState() { return LZ4_STREAMSIZE; } + + + +/******************************** +* Compression functions +********************************/ + +static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +static const U64 prime5bytes = 889523592379ULL; +static U32 LZ4_hashSequence64(size_t sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + const U32 hashMask = (1<> (40 - hashLog)) & hashMask; +} + +static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType) +{ + if (LZ4_64bits()) + return LZ4_hashSequence64(sequence, tableType); + return LZ4_hashSequence((U32)sequence, tableType); +} + +static U32 LZ4_hashPosition(const void* p, tableType_t tableType) { return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); } + +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +{ + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } +} + +static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ +} + +static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +} + +FORCE_INLINE int LZ4_compress_generic( + void* const ctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) +{ + LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; + + const BYTE* ip = (const BYTE*) source; + const BYTE* base; + const BYTE* lowLimit; + const BYTE* const lowRefLimit = ip - dictPtr->dictSize; + const BYTE* const dictionary = dictPtr->dictionary; + const BYTE* const dictEnd = dictionary + dictPtr->dictSize; + const size_t dictDelta = dictEnd - (const BYTE*)source; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 forwardH; + size_t refDelta=0; + + /* Init conditions */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + switch(dict) + { + case noDict: + default: + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; + case withPrefix64k: + base = (const BYTE*)source - dictPtr->currentOffset; + lowLimit = (const BYTE*)source - dictPtr->dictSize; + break; + case usingExtDict: + base = (const BYTE*)source - dictPtr->currentOffset; + lowLimit = (const BYTE*)source; + break; + } + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (inputSize> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, ctx, tableType, base); + if (dict==usingExtDict) + { + if (match<(const BYTE*)source) + { + refDelta = dictDelta; + lowLimit = dictionary; + } + else + { + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + + { + /* Encode Literal length */ + unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; /* Check output limit */ + if (litLength>=RUN_MASK) + { + int len = (int)litLength-RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchLength; + if (ip==limit) + { + unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchLength += more; + ip += more; + } + } + else + { + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchLength; + } + + if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) + return 0; /* Check output limit */ + if (matchLength>=ML_MASK) + { + *token += ML_MASK; + matchLength -= ML_MASK; + for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; } + if (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, ctx, tableType, base); + if (dict==usingExtDict) + { + if (match<(const BYTE*)source) + { + refDelta = dictDelta; + lowLimit = dictionary; + } + else + { + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } + LZ4_putPosition(ip, ctx, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { + const size_t lastRun = (size_t)(iend - anchor); + if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) + return 0; /* Check output limit */ + if (lastRun >= RUN_MASK) + { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRun<= LZ4_compressBound(inputSize)) + { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } + else + { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } +} + + +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ +#if (HEAPMODE) + void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ4_stream_t ctx; + void* ctxPtr = &ctx; +#endif + + int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + +#if (HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + + +int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); +} + + +/* hidden debug function */ +/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ +int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t ctx; + + LZ4_resetStream(&ctx); + + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); +} + + +/******************************** +* destSize variant +********************************/ + +static int LZ4_compress_destSize_generic( + void* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + const int targetDstSize, + const tableType_t tableType) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* base = (const BYTE*) src; + const BYTE* lowLimit = (const BYTE*) src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + targetDstSize; + BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; + BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); + BYTE* const oMaxSeq = oMaxLit - 1 /* token */; + + U32 forwardH; + + + /* Init conditions */ + if (targetDstSize < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (*srcSizePtr> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) + goto _last_literals; + + match = LZ4_getPositionOnHash(h, ctx, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + { + /* Encode Literal length */ + unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if (op + ((litLength+240)/255) + litLength > oMaxLit) + { + /* Not enough space for a last match */ + op--; + goto _last_literals; + } + if (litLength>=RUN_MASK) + { + unsigned len = litLength - RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< oMaxMatch) + { + /* Match description too long : reduce it */ + matchLength = (15-1) + (oMaxMatch-op) * 255; + } + //printf("offset %5i, matchLength%5i \n", (int)(ip-match), matchLength + MINMATCH); + ip += MINMATCH + matchLength; + + if (matchLength>=ML_MASK) + { + *token += ML_MASK; + matchLength -= ML_MASK; + while (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of block */ + if (ip > mflimit) break; + if (op > oMaxSeq) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, ctx, tableType, base); + if ( (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { + size_t lastRunSize = (size_t)(iend - anchor); + if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) + { + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (oend-op) - 1; + lastRunSize -= (lastRunSize+240)/255; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) + { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) /* compression success is guaranteed */ + { + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } + else + { + if (*srcSizePtr < LZ4_64Klimit) + return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16); + else + return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); + } +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (HEAPMODE) + void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ4_stream_t ctxBody; + void* ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); + +#if (HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + + +/******************************** +* Streaming functions +********************************/ + +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_resetStream(lz4s); + return lz4s; +} + +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); +} + +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + FREEMEM(LZ4_stream); + return (0); +} + + +#define HASH_UNIT sizeof(size_t) +int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ + LZ4_resetStream(LZ4_dict); + + if (dictSize < (int)HASH_UNIT) + { + dict->dictionary = NULL; + dict->dictSize = 0; + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->currentOffset += 64 KB; + base = p - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->currentOffset += dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) + { + LZ4_putPosition(p, dict->hashTable, byU32, base); + p+=3; + } + + return dict->dictSize; +} + + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +{ + if ((LZ4_dict->currentOffset > 0x80000000) || + ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */ + { + /* rescale hash table */ + U32 delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = (const BYTE*) source; + if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; + LZ4_renormDictT(streamPtr, smallest); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) + { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == (const BYTE*)source) + { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + else + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + streamPtr->dictSize += (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } + + /* external dictionary mode */ + { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + else + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } +} + + +/* Hidden debug function, to force external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +{ + LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; + int result; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = dictEnd; + if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; + LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); + + result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + + return result; +} + + +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; + + if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/******************************* +* Decompression functions +*******************************/ +/* + * This generic decompression function cover all use cases. + * It shall be instantiated several times, using different sets of directives + * Note that it is essential this generic function is really inlined, + * in order to remove useless branches during compilation optimization. + */ +FORCE_INLINE int LZ4_decompress_generic( + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest if dict == noDict */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + /* Local Variables */ + const BYTE* ip = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; + const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; + + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + + + /* Special cases */ + if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + + + /* Main Loop */ + while (1) + { + unsigned token; + size_t length; + const BYTE* match; + + /* get literal length */ + token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) + { + unsigned s; + do + { + s = *ip++; + length += s; + } + while (likely((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-COPYLENGTH))) + { + if (partialDecoding) + { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } + else + { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ + match = cpy - LZ4_readLE16(ip); ip+=2; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) + { + unsigned s; + do + { + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + s = *ip++; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) + { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) + { + /* match can be copied as a single segment from external dictionary */ + match = dictEnd - (lowPrefix-match); + memmove(op, match, length); op += length; + } + else + { + /* match encompass external dictionary and current segment */ + size_t copySize = (size_t)(lowPrefix-match); + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + copySize = length - copySize; + if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */ + { + BYTE* const endOfMatch = op + copySize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } + else + { + memcpy(op, lowPrefix, copySize); + op += copySize; + } + } + continue; + } + + /* copy repeated sequence */ + cpy = op + length; + if (unlikely((op-match)<8)) + { + const size_t dec64 = dec64table[op-match]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[op-match]; + LZ4_copy4(op+4, match); + op += 8; match -= dec64; + } else { LZ4_copy8(op, match); op+=8; match+=8; } + + if (unlikely(cpy>oend-12)) + { + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */ + if (op < oend-8) + { + LZ4_wildCopy(op, match, oend-8); + match += (oend-8) - op; + op = oend-8; + } + while (opprefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) + { + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += result; + lz4sd->prefixEnd += result; + } + else + { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) + { + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += originalSize; + lz4sd->prefixEnd += originalSize; + } + else + { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) + { + if (dictSize >= (int)(64 KB - 1)) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); +} + +/* debug function */ +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + + +/*************************************************** +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } +int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } + +/* +These function names are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } + + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } + +static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base) +{ + MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); + lz4ds->bufferStart = base; +} + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ + LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer); + return 0; +} + +void* LZ4_create (char* inputBuffer) +{ + void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer); + return lz4ds; +} + +char* LZ4_slideInputBuffer (void* LZ4_Data) +{ + LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data; + int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); + return (char*)(ctx->bufferStart + dictSize); +} + +/* Obsolete streaming decompression functions */ + +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} + +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} + +#endif /* LZ4_COMMONDEFS_ONLY */ + diff --git a/target/bitshuffle-0.3.4/lz4/lz4.h b/target/bitshuffle-0.3.4/lz4/lz4.h new file mode 100644 index 00000000..3e740022 --- /dev/null +++ b/target/bitshuffle-0.3.4/lz4/lz4.h @@ -0,0 +1,360 @@ +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/* + * lz4.h provides block compression functions, and gives full buffer control to programmer. + * If you need to generate inter-operable compressed data (respecting LZ4 frame specification), + * and can let the library handle its own memory, please use lz4frame.h instead. +*/ + +/************************************** +* Version +**************************************/ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) +int LZ4_versionNumber (void); + +/************************************** +* Tuning parameter +**************************************/ +/* + * LZ4_MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache + */ +#define LZ4_MEMORY_USAGE 14 + + +/************************************** +* Simple Functions +**************************************/ + +int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); +int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + +/* +LZ4_compress_default() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails + +LZ4_decompress_safe() : + compressedSize : is the precise full size of the compressed block. + maxDecompressedSize : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) + If destination buffer is not large enough, decoding will stop and output an error code (<0). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside output buffer, nor reads outside input buffer. +*/ + + +/************************************** +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/* +LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) +*/ +int LZ4_compressBound(int inputSize); + +/* +LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows to select an "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. +*/ +int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); + + +/* +LZ4_compress_fast_extState() : + Same compression function, just using an externally allocated memory space to store compression state. + Use LZ4_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. +*/ +int LZ4_sizeofState(void); +int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); + + +/* +LZ4_compress_destSize() : + Reverse the logic, by compressing as much data as possible from 'source' buffer + into already allocated buffer 'dest' of size 'targetDestSize'. + This function either compresses the entire 'source' content into 'dest' if it's large enough, + or fill 'dest' buffer completely with as much data as possible from 'source'. + *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. + New value is necessarily <= old value. + return : Nb bytes written into 'dest' (necessarily <= targetDestSize) + or 0 if compression fails +*/ +int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); + + +/* +LZ4_decompress_fast() : + originalSize : is the original and therefore uncompressed size + return : the number of bytes read from the source buffer (in other words, the compressed size) + If the source stream is detected malformed, the function will stop decoding and return a negative result. + Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. + note : This function fully respect memory boundaries for properly formed compressed data. + It is a bit faster than LZ4_decompress_safe(). + However, it does not provide any protection against intentionally modified data stream (malicious input). + Use this function in trusted environment only (data to decode comes from a trusted source). +*/ +int LZ4_decompress_fast (const char* source, char* dest, int originalSize); + +/* +LZ4_decompress_safe_partial() : + This function decompress a compressed block of size 'compressedSize' at position 'source' + into destination buffer 'dest' of size 'maxDecompressedSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +*/ +int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); + + +/*********************************************** +* Streaming Compression Functions +***********************************************/ +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) +/* + * LZ4_stream_t + * information structure to track an LZ4 stream. + * important : init this structure content before first use ! + * note : only allocated directly the structure if you are statically linking LZ4 + * If you are using liblz4 as a DLL, please use below construction methods instead. + */ +typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; + +/* + * LZ4_resetStream + * Use this function to init an allocated LZ4_stream_t structure + */ +void LZ4_resetStream (LZ4_stream_t* streamPtr); + +/* + * LZ4_createStream will allocate and initialize an LZ4_stream_t structure + * LZ4_freeStream releases its memory. + * In the context of a DLL (liblz4), please use these methods rather than the static struct. + * They are more future proof, in case of a change of LZ4_stream_t size. + */ +LZ4_stream_t* LZ4_createStream(void); +int LZ4_freeStream (LZ4_stream_t* streamPtr); + +/* + * LZ4_loadDict + * Use this function to load a static dictionary into LZ4_stream. + * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Loading a size of 0 is allowed. + * Return : dictionary size, in bytes (necessarily <= 64 KB) + */ +int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/* + * LZ4_compress_fast_continue + * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * 'dst' buffer must be already allocated. + * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + */ +int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); + +/* + * LZ4_saveDict + * If previously compressed data block is not guaranteed to remain available at its memory location + * save it into a safer place (char* safeBuffer) + * Note : you don't need to call LZ4_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue() + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error + */ +int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); + + +/************************************************ +* Streaming Decompression Functions +************************************************/ + +#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; +/* + * LZ4_streamDecode_t + * information structure to track an LZ4 stream. + * init this structure content using LZ4_setStreamDecode or memset() before first use ! + * + * In the context of a DLL (liblz4) please prefer usage of construction methods below. + * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future. + * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure + * LZ4_freeStreamDecode releases its memory. + */ +LZ4_streamDecode_t* LZ4_createStreamDecode(void); +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); + +/* + * LZ4_setStreamDecode + * Use this function to instruct where to find the dictionary. + * Setting a size of 0 is allowed (same effect as reset). + * Return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ4_setStreamDecode() +*/ +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as + a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue() + They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure. +*/ +int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); +int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); + + + +/************************************** +* Obsolete Functions +**************************************/ +/* Deprecate Warnings */ +/* Should these warnings messages be a problem, + it is generally possible to disable them, + with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual for example. + You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ +#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ4_GCC_VERSION >= 301) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# define LZ4_DEPRECATED(message) +# endif +#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */ + +/* Obsolete compression functions */ +/* These functions are planned to start generate warnings by r131 approximately */ +int LZ4_compress (const char* source, char* dest, int sourceSize); +int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Obsolete decompression functions */ +/* These function names are completely deprecated and must no longer be used. + They are only provided here for compatibility with older programs. + - LZ4_uncompress is the same as LZ4_decompress_fast + - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe + These function prototypes are now disabled; uncomment them only if you really need them. + It is highly recommended to stop using these prototypes and migrate to maintained ones */ +/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ +/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ + +/* Obsolete streaming functions; use new streaming interface whenever possible */ +LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); + +/* Obsolete streaming decoding functions */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + + +#if defined (__cplusplus) +} +#endif diff --git a/target/bitshuffle-0.3.4/lzf/LICENSE.txt b/target/bitshuffle-0.3.4/lzf/LICENSE.txt new file mode 100644 index 00000000..3787a007 --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/LICENSE.txt @@ -0,0 +1,34 @@ +Copyright Notice and Statement for LZF filter + +Copyright (c) 2008-2009 Andrew Collette +http://h5py.alfven.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +a. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +c. Neither the name of the author nor the names of contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/target/bitshuffle-0.3.4/lzf/README.txt b/target/bitshuffle-0.3.4/lzf/README.txt new file mode 100644 index 00000000..c6ad62c3 --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/README.txt @@ -0,0 +1,84 @@ +=============================== +LZF filter for HDF5, revision 3 +=============================== + +The LZF filter provides high-speed compression with acceptable compression +performance, resulting in much faster performance than DEFLATE, at the +cost of a slightly lower compression ratio. It's appropriate for large +datasets of low to moderate complexity, for which some compression is +much better than none, but for which the speed of DEFLATE is unacceptable. + +This filter has been tested against HDF5 versions 1.6.5 through 1.8.3. It +is released under the BSD license (see LICENSE.txt for details). + + +Using the filter from HDF5 +-------------------------- + +There is exactly one new public function declared in lzf_filter.h, with +the following signature: + + int register_lzf(void) + +Calling this will register the filter with the HDF5 library. A non-negative +return value indicates success. If the registration fails, an error is pushed +onto the current error stack and a negative value is returned. + +It's strongly recommended to use the SHUFFLE filter with LZF, as it's +cheap, supported by all current versions of HDF5, and can significantly +improve the compression ratio. An example C program ("example.c") is included +which demonstrates the proper use of the filter. + + +Compiling +--------- + +The filter consists of a single .c file and header, along with an embedded +version of the LZF compression library. Since the filter is stateless, it's +recommended to statically link the entire thing into your program; for +example: + + $ gcc -O2 -lhdf5 lzf/*.c lzf_filter.c myprog.c -o myprog + +It can also be built as a shared library, although you will have to install +the resulting library somewhere the runtime linker can find it: + + $ gcc -O2 -lhdf5 -fPIC -shared lzf/*.c lzf_filter.c -o liblzf_filter.so + +A similar procedure should be used for building C++ code. As in these +examples, using option -O1 or higher is strongly recommended for increased +performance. + + +Contact +------- + +This filter is maintained as part of the HDF5 for Python (h5py) project. The +goal of h5py is to provide access to the majority of the HDF5 C API and feature +set from Python. The most recent version of h5py (1.1) includes the LZF +filter by default. + +* Downloads and bug tracker: http://h5py.googlecode.com + +* Main web site and documentation: http://h5py.alfven.org + +* Contact email: h5py at alfven dot org + + +History of changes +------------------ + +Revision 3 (6/25/09) + Fix issue with changed filter struct definition under HDF5 1.8.3. + +Revision 2 + Minor speed enhancement. + +Revision 1 + Initial release. + + + + + + diff --git a/target/bitshuffle-0.3.4/lzf/README_bitshuffle.txt b/target/bitshuffle-0.3.4/lzf/README_bitshuffle.txt new file mode 100644 index 00000000..d620a925 --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/README_bitshuffle.txt @@ -0,0 +1,3 @@ +The LZF filter for HDF5 is part of the h5py project (http://h5py.alfven.org). +The version included with bitshuffle is from version 2.3 of h5py with no +modifications other than the addition of this README. diff --git a/target/bitshuffle-0.3.4/lzf/example.c b/target/bitshuffle-0.3.4/lzf/example.c new file mode 100644 index 00000000..23dd776c --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/example.c @@ -0,0 +1,106 @@ +/* + Copyright (C) 2009 Andrew Collette + http://h5py.alfven.org + License: BSD (see LICENSE.txt) + + Example program demonstrating use of the LZF filter from C code. + + To compile this program: + + h5cc -DH5_USE_16_API lzf/*.c lzf_filter.c example.c -o example + + To run: + + $ ./example + Success! + $ h5ls -v test_lzf.hdf5 + Opened "test_lzf.hdf5" with sec2 driver. + dset Dataset {100/100, 100/100, 100/100} + Location: 0:1:0:976 + Links: 1 + Modified: 2009-02-15 16:35:11 PST + Chunks: {1, 100, 100} 40000 bytes + Storage: 4000000 logical bytes, 174288 allocated bytes, 2295.05% utilization + Filter-0: shuffle-2 OPT {4} + Filter-1: lzf-32000 OPT {1, 261, 40000} + Type: native float +*/ + +#include +#include "hdf5.h" +#include "lzf_filter.h" + +#define SIZE 100*100*100 +#define SHAPE {100,100,100} +#define CHUNKSHAPE {1,100,100} + +int main(){ + + static float data[SIZE]; + static float data_out[SIZE]; + const hsize_t shape[] = SHAPE; + const hsize_t chunkshape[] = CHUNKSHAPE; + int r, i; + int return_code = 1; + + hid_t fid, sid, dset, plist = 0; + + for(i=0; i0) H5Dclose(dset); + if(sid>0) H5Sclose(sid); + if(plist>0) H5Pclose(plist); + if(fid>0) H5Fclose(fid); + + return return_code; +} + diff --git a/target/bitshuffle-0.3.4/lzf/lzf/lzf.h b/target/bitshuffle-0.3.4/lzf/lzf/lzf.h new file mode 100644 index 00000000..919b6e6b --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/lzf/lzf.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2000-2008 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License ("GPL") version 2 or any later version, + * in which case the provisions of the GPL are applicable instead of + * the above. If you wish to allow the use of your version of this file + * only under the terms of the GPL and not to allow others to use your + * version of this file under the BSD license, indicate your decision + * by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL. If you do not delete the + * provisions above, a recipient may use your version of this file under + * either the BSD or the GPL. + */ + +#ifndef LZF_H +#define LZF_H + +/*********************************************************************** +** +** lzf -- an extremely fast/free compression/decompression-method +** http://liblzf.plan9.de/ +** +** This algorithm is believed to be patent-free. +** +***********************************************************************/ + +#define LZF_VERSION 0x0105 /* 1.5, API version */ + +/* + * Compress in_len bytes stored at the memory block starting at + * in_data and write the result to out_data, up to a maximum length + * of out_len bytes. + * + * If the output buffer is not large enough or any error occurs return 0, + * otherwise return the number of bytes used, which might be considerably + * more than in_len (but less than 104% of the original size), so it + * makes sense to always use out_len == in_len - 1), to ensure _some_ + * compression, and store the data uncompressed otherwise (with a flag, of + * course. + * + * lzf_compress might use different algorithms on different systems and + * even different runs, thus might result in different compressed strings + * depending on the phase of the moon or similar factors. However, all + * these strings are architecture-independent and will result in the + * original data when decompressed using lzf_decompress. + * + * The buffers must not be overlapping. + * + * If the option LZF_STATE_ARG is enabled, an extra argument must be + * supplied which is not reflected in this header file. Refer to lzfP.h + * and lzf_c.c. + * + */ +unsigned int +lzf_compress (const void *const in_data, unsigned int in_len, + void *out_data, unsigned int out_len); + +/* + * Decompress data compressed with some version of the lzf_compress + * function and stored at location in_data and length in_len. The result + * will be stored at out_data up to a maximum of out_len characters. + * + * If the output buffer is not large enough to hold the decompressed + * data, a 0 is returned and errno is set to E2BIG. Otherwise the number + * of decompressed bytes (i.e. the original length of the data) is + * returned. + * + * If an error in the compressed data is detected, a zero is returned and + * errno is set to EINVAL. + * + * This function is very fast, about as fast as a copying loop. + */ +unsigned int +lzf_decompress (const void *const in_data, unsigned int in_len, + void *out_data, unsigned int out_len); + +#endif + diff --git a/target/bitshuffle-0.3.4/lzf/lzf/lzfP.h b/target/bitshuffle-0.3.4/lzf/lzf/lzfP.h new file mode 100644 index 00000000..8414da4d --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/lzf/lzfP.h @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2000-2007 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License ("GPL") version 2 or any later version, + * in which case the provisions of the GPL are applicable instead of + * the above. If you wish to allow the use of your version of this file + * only under the terms of the GPL and not to allow others to use your + * version of this file under the BSD license, indicate your decision + * by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL. If you do not delete the + * provisions above, a recipient may use your version of this file under + * either the BSD or the GPL. + */ + +#ifndef LZFP_h +#define LZFP_h + +#define STANDALONE 1 /* at the moment, this is ok. */ + +#ifndef STANDALONE +# include "lzf.h" +#endif + +/* + * Size of hashtable is (1 << HLOG) * sizeof (char *) + * decompression is independent of the hash table size + * the difference between 15 and 14 is very small + * for small blocks (and 14 is usually a bit faster). + * For a low-memory/faster configuration, use HLOG == 13; + * For best compression, use 15 or 16 (or more, up to 23). + */ +#ifndef HLOG +# define HLOG 17 /* Avoid pathological case at HLOG=16 A.C. 2/15/09 */ +#endif + +/* + * Sacrifice very little compression quality in favour of compression speed. + * This gives almost the same compression as the default code, and is + * (very roughly) 15% faster. This is the preferred mode of operation. + */ +#ifndef VERY_FAST +# define VERY_FAST 1 +#endif + +/* + * Sacrifice some more compression quality in favour of compression speed. + * (roughly 1-2% worse compression for large blocks and + * 9-10% for small, redundant, blocks and >>20% better speed in both cases) + * In short: when in need for speed, enable this for binary data, + * possibly disable this for text data. + */ +#ifndef ULTRA_FAST +# define ULTRA_FAST 1 +#endif + +/* + * Unconditionally aligning does not cost very much, so do it if unsure + */ +#ifndef STRICT_ALIGN +# define STRICT_ALIGN !(defined(__i386) || defined (__amd64)) +#endif + +/* + * You may choose to pre-set the hash table (might be faster on some + * modern cpus and large (>>64k) blocks, and also makes compression + * deterministic/repeatable when the configuration otherwise is the same). + */ +#ifndef INIT_HTAB +# define INIT_HTAB 0 +#endif + +/* ======================================================================= + Changing things below this line may break the HDF5 LZF filter. + A.C. 2/15/09 + ======================================================================= +*/ + +/* + * Avoid assigning values to errno variable? for some embedding purposes + * (linux kernel for example), this is neccessary. NOTE: this breaks + * the documentation in lzf.h. + */ +#ifndef AVOID_ERRNO +# define AVOID_ERRNO 0 +#endif + +/* + * Wether to pass the LZF_STATE variable as argument, or allocate it + * on the stack. For small-stack environments, define this to 1. + * NOTE: this breaks the prototype in lzf.h. + */ +#ifndef LZF_STATE_ARG +# define LZF_STATE_ARG 0 +#endif + +/* + * Wether to add extra checks for input validity in lzf_decompress + * and return EINVAL if the input stream has been corrupted. This + * only shields against overflowing the input buffer and will not + * detect most corrupted streams. + * This check is not normally noticable on modern hardware + * (<1% slowdown), but might slow down older cpus considerably. + */ + +#ifndef CHECK_INPUT +# define CHECK_INPUT 1 +#endif + +/*****************************************************************************/ +/* nothing should be changed below */ + +typedef unsigned char u8; + +typedef const u8 *LZF_STATE[1 << (HLOG)]; + +#if !STRICT_ALIGN +/* for unaligned accesses we need a 16 bit datatype. */ +# include +# if USHRT_MAX == 65535 + typedef unsigned short u16; +# elif UINT_MAX == 65535 + typedef unsigned int u16; +# else +# undef STRICT_ALIGN +# define STRICT_ALIGN 1 +# endif +#endif + +#if ULTRA_FAST +# if defined(VERY_FAST) +# undef VERY_FAST +# endif +#endif + +#if INIT_HTAB +# ifdef __cplusplus +# include +# else +# include +# endif +#endif + +#endif + diff --git a/target/bitshuffle-0.3.4/lzf/lzf/lzf_c.c b/target/bitshuffle-0.3.4/lzf/lzf/lzf_c.c new file mode 100644 index 00000000..fbfd4cce --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/lzf/lzf_c.c @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2000-2008 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License ("GPL") version 2 or any later version, + * in which case the provisions of the GPL are applicable instead of + * the above. If you wish to allow the use of your version of this file + * only under the terms of the GPL and not to allow others to use your + * version of this file under the BSD license, indicate your decision + * by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL. If you do not delete the + * provisions above, a recipient may use your version of this file under + * either the BSD or the GPL. + */ + +#include "lzfP.h" + +#define HSIZE (1 << (HLOG)) + +/* + * don't play with this unless you benchmark! + * decompression is not dependent on the hash function + * the hashing function might seem strange, just believe me + * it works ;) + */ +#ifndef FRST +# define FRST(p) (((p[0]) << 8) | p[1]) +# define NEXT(v,p) (((v) << 8) | p[2]) +# if ULTRA_FAST +# define IDX(h) ((( h >> (3*8 - HLOG)) - h ) & (HSIZE - 1)) +# elif VERY_FAST +# define IDX(h) ((( h >> (3*8 - HLOG)) - h*5) & (HSIZE - 1)) +# else +# define IDX(h) ((((h ^ (h << 5)) >> (3*8 - HLOG)) - h*5) & (HSIZE - 1)) +# endif +#endif +/* + * IDX works because it is very similar to a multiplicative hash, e.g. + * ((h * 57321 >> (3*8 - HLOG)) & (HSIZE - 1)) + * the latter is also quite fast on newer CPUs, and compresses similarly. + * + * the next one is also quite good, albeit slow ;) + * (int)(cos(h & 0xffffff) * 1e6) + */ + +#if 0 +/* original lzv-like hash function, much worse and thus slower */ +# define FRST(p) (p[0] << 5) ^ p[1] +# define NEXT(v,p) ((v) << 5) ^ p[2] +# define IDX(h) ((h) & (HSIZE - 1)) +#endif + +#define MAX_LIT (1 << 5) +#define MAX_OFF (1 << 13) +#define MAX_REF ((1 << 8) + (1 << 3)) + +#if __GNUC__ >= 3 +# define expect(expr,value) __builtin_expect ((expr),(value)) +# define inline inline +#else +# define expect(expr,value) (expr) +# define inline static +#endif + +#define expect_false(expr) expect ((expr) != 0, 0) +#define expect_true(expr) expect ((expr) != 0, 1) + +/* + * compressed format + * + * 000LLLLL ; literal + * LLLooooo oooooooo ; backref L + * 111ooooo LLLLLLLL oooooooo ; backref L+7 + * + */ + +unsigned int +lzf_compress (const void *const in_data, unsigned int in_len, + void *out_data, unsigned int out_len +#if LZF_STATE_ARG + , LZF_STATE htab +#endif + ) +{ +#if !LZF_STATE_ARG + LZF_STATE htab; +#endif + const u8 **hslot; + const u8 *ip = (const u8 *)in_data; + u8 *op = (u8 *)out_data; + const u8 *in_end = ip + in_len; + u8 *out_end = op + out_len; + const u8 *ref; + + /* off requires a type wide enough to hold a general pointer difference. + * ISO C doesn't have that (size_t might not be enough and ptrdiff_t only + * works for differences within a single object). We also assume that no + * no bit pattern traps. Since the only platform that is both non-POSIX + * and fails to support both assumptions is windows 64 bit, we make a + * special workaround for it. + */ +#if ( defined (WIN32) && defined (_M_X64) ) || defined (_WIN64) + unsigned _int64 off; /* workaround for missing POSIX compliance */ +#else + unsigned long off; +#endif + unsigned int hval; + int lit; + + if (!in_len || !out_len) + return 0; + +#if INIT_HTAB + memset (htab, 0, sizeof (htab)); +# if 0 + for (hslot = htab; hslot < htab + HSIZE; hslot++) + *hslot++ = ip; +# endif +#endif + + lit = 0; op++; /* start run */ + + hval = FRST (ip); + while (ip < in_end - 2) + { + hval = NEXT (hval, ip); + hslot = htab + IDX (hval); + ref = *hslot; *hslot = ip; + + if (1 +#if INIT_HTAB + && ref < ip /* the next test will actually take care of this, but this is faster */ +#endif + && (off = ip - ref - 1) < MAX_OFF + && ip + 4 < in_end + && ref > (u8 *)in_data +#if STRICT_ALIGN + && ref[0] == ip[0] + && ref[1] == ip[1] + && ref[2] == ip[2] +#else + && *(u16 *)ref == *(u16 *)ip + && ref[2] == ip[2] +#endif + ) + { + /* match found at *ref++ */ + unsigned int len = 2; + unsigned int maxlen = in_end - ip - len; + maxlen = maxlen > MAX_REF ? MAX_REF : maxlen; + + if (expect_false (op + 3 + 1 >= out_end)) /* first a faster conservative test */ + if (op - !lit + 3 + 1 >= out_end) /* second the exact but rare test */ + return 0; + + op [- lit - 1] = lit - 1; /* stop run */ + op -= !lit; /* undo run if length is zero */ + + for (;;) + { + if (expect_true (maxlen > 16)) + { + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + len++; if (ref [len] != ip [len]) break; + } + + do + len++; + while (len < maxlen && ref[len] == ip[len]); + + break; + } + + len -= 2; /* len is now #octets - 1 */ + ip++; + + if (len < 7) + { + *op++ = (off >> 8) + (len << 5); + } + else + { + *op++ = (off >> 8) + ( 7 << 5); + *op++ = len - 7; + } + + *op++ = off; + lit = 0; op++; /* start run */ + + ip += len + 1; + + if (expect_false (ip >= in_end - 2)) + break; + +#if ULTRA_FAST || VERY_FAST + --ip; +# if VERY_FAST && !ULTRA_FAST + --ip; +# endif + hval = FRST (ip); + + hval = NEXT (hval, ip); + htab[IDX (hval)] = ip; + ip++; + +# if VERY_FAST && !ULTRA_FAST + hval = NEXT (hval, ip); + htab[IDX (hval)] = ip; + ip++; +# endif +#else + ip -= len + 1; + + do + { + hval = NEXT (hval, ip); + htab[IDX (hval)] = ip; + ip++; + } + while (len--); +#endif + } + else + { + /* one more literal byte we must copy */ + if (expect_false (op >= out_end)) + return 0; + + lit++; *op++ = *ip++; + + if (expect_false (lit == MAX_LIT)) + { + op [- lit - 1] = lit - 1; /* stop run */ + lit = 0; op++; /* start run */ + } + } + } + + if (op + 3 > out_end) /* at most 3 bytes can be missing here */ + return 0; + + while (ip < in_end) + { + lit++; *op++ = *ip++; + + if (expect_false (lit == MAX_LIT)) + { + op [- lit - 1] = lit - 1; /* stop run */ + lit = 0; op++; /* start run */ + } + } + + op [- lit - 1] = lit - 1; /* end run */ + op -= !lit; /* undo run if length is zero */ + + return op - (u8 *)out_data; +} + diff --git a/target/bitshuffle-0.3.4/lzf/lzf/lzf_d.c b/target/bitshuffle-0.3.4/lzf/lzf/lzf_d.c new file mode 100644 index 00000000..2e2eedaa --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/lzf/lzf_d.c @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2000-2007 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License ("GPL") version 2 or any later version, + * in which case the provisions of the GPL are applicable instead of + * the above. If you wish to allow the use of your version of this file + * only under the terms of the GPL and not to allow others to use your + * version of this file under the BSD license, indicate your decision + * by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL. If you do not delete the + * provisions above, a recipient may use your version of this file under + * either the BSD or the GPL. + */ + +#include "lzfP.h" + +#if AVOID_ERRNO +# define SET_ERRNO(n) +#else +# include +# define SET_ERRNO(n) errno = (n) +#endif + +/* ASM is slower than C in HDF5 tests -- A.C. 2/5/09 +#ifndef __STRICT_ANSI__ +#ifndef H5PY_DISABLE_LZF_ASM +#if (__i386 || __amd64) && __GNUC__ >= 3 +# define lzf_movsb(dst, src, len) \ + asm ("rep movsb" \ + : "=D" (dst), "=S" (src), "=c" (len) \ + : "0" (dst), "1" (src), "2" (len)); +#endif +#endif +#endif +*/ + +unsigned int +lzf_decompress (const void *const in_data, unsigned int in_len, + void *out_data, unsigned int out_len) +{ + u8 const *ip = (const u8 *)in_data; + u8 *op = (u8 *)out_data; + u8 const *const in_end = ip + in_len; + u8 *const out_end = op + out_len; + + do + { + unsigned int ctrl = *ip++; + + if (ctrl < (1 << 5)) /* literal run */ + { + ctrl++; + + if (op + ctrl > out_end) + { + SET_ERRNO (E2BIG); + return 0; + } + +#if CHECK_INPUT + if (ip + ctrl > in_end) + { + SET_ERRNO (EINVAL); + return 0; + } +#endif + +#ifdef lzf_movsb + lzf_movsb (op, ip, ctrl); +#else + do + *op++ = *ip++; + while (--ctrl); +#endif + } + else /* back reference */ + { + unsigned int len = ctrl >> 5; + + u8 *ref = op - ((ctrl & 0x1f) << 8) - 1; + +#if CHECK_INPUT + if (ip >= in_end) + { + SET_ERRNO (EINVAL); + return 0; + } +#endif + if (len == 7) + { + len += *ip++; +#if CHECK_INPUT + if (ip >= in_end) + { + SET_ERRNO (EINVAL); + return 0; + } +#endif + } + + ref -= *ip++; + + if (op + len + 2 > out_end) + { + SET_ERRNO (E2BIG); + return 0; + } + + if (ref < (u8 *)out_data) + { + SET_ERRNO (EINVAL); + return 0; + } + +#ifdef lzf_movsb + len += 2; + lzf_movsb (op, ref, len); +#else + *op++ = *ref++; + *op++ = *ref++; + + do + *op++ = *ref++; + while (--len); +#endif + } + } + while (ip < in_end); + + return op - (u8 *)out_data; +} + diff --git a/target/bitshuffle-0.3.4/lzf/lzf_filter.c b/target/bitshuffle-0.3.4/lzf/lzf_filter.c new file mode 100644 index 00000000..c6dd4b0e --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/lzf_filter.c @@ -0,0 +1,261 @@ +/***** Preamble block ********************************************************* +* +* This file is part of h5py, a low-level Python interface to the HDF5 library. +* +* Copyright (C) 2008 Andrew Collette +* http://h5py.alfven.org +* License: BSD (See LICENSE.txt for full license) +* +* $Date$ +* +****** End preamble block ****************************************************/ + +/* + Implements an LZF filter module for HDF5, using the BSD-licensed library + by Marc Alexander Lehmann (http://www.goof.com/pcg/marc/liblzf.html). + + No Python-specific code is used. The filter behaves like the DEFLATE + filter, in that it is called for every type and space, and returns 0 + if the data cannot be compressed. + + The only public function is (int) register_lzf(void), which passes on + the result from H5Zregister. +*/ + +#include +#include +#include +#include "hdf5.h" +#include "lzf/lzf.h" +#include "lzf_filter.h" + +/* Our own versions of H5Epush_sim, as it changed in 1.8 */ +#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 7 + +#define PUSH_ERR(func, minor, str) H5Epush(__FILE__, func, __LINE__, H5E_PLINE, minor, str) +#define H5PY_GET_FILTER H5Pget_filter_by_id + +#else + +#define PUSH_ERR(func, minor, str) H5Epush1(__FILE__, func, __LINE__, H5E_PLINE, minor, str) +#define H5PY_GET_FILTER(a,b,c,d,e,f,g) H5Pget_filter_by_id2(a,b,c,d,e,f,g,NULL) + +#endif + +/* Deal with the mutiple definitions for H5Z_class_t. + Note: Only HDF5 1.6 and 1.8 are supported. + + (1) The old class should always be used for HDF5 1.6 + (2) The new class should always be used for HDF5 1.8 < 1.8.3 + (3) The old class should be used for HDF5 1.8 >= 1.8.3 only if the + macro H5_USE_16_API is set +*/ + +#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 8 && (H5_VERS_RELEASE < 3 || !H5_USE_16_API) +#define H5PY_H5Z_NEWCLS 1 +#else +#define H5PY_H5Z_NEWCLS 0 +#endif + +size_t lzf_filter(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + +herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space); + + +/* Try to register the filter, passing on the HDF5 return value */ +int register_lzf(void){ + + int retval; + +#if H5PY_H5Z_NEWCLS + H5Z_class_t filter_class = { + H5Z_CLASS_T_VERS, + (H5Z_filter_t)(H5PY_FILTER_LZF), + 1, 1, + "lzf", + NULL, + (H5Z_set_local_func_t)(lzf_set_local), + (H5Z_func_t)(lzf_filter) + }; +#else + H5Z_class_t filter_class = { + (H5Z_filter_t)(H5PY_FILTER_LZF), + "lzf", + NULL, + (H5Z_set_local_func_t)(lzf_set_local), + (H5Z_func_t)(lzf_filter) + }; +#endif + + retval = H5Zregister(&filter_class); + if(retval<0){ + PUSH_ERR("register_lzf", H5E_CANTREGISTER, "Can't register LZF filter"); + } + return retval; +} + +/* Filter setup. Records the following inside the DCPL: + + 1. If version information is not present, set slots 0 and 1 to the filter + revision and LZF API version, respectively. + + 2. Compute the chunk size in bytes and store it in slot 2. +*/ +herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space){ + + int ndims; + int i; + herr_t r; + + unsigned int bufsize; + hsize_t chunkdims[32]; + + unsigned int flags; + size_t nelements = 8; + unsigned values[] = {0,0,0,0,0,0,0,0}; + + r = H5PY_GET_FILTER(dcpl, H5PY_FILTER_LZF, &flags, &nelements, values, 0, NULL); + if(r<0) return -1; + + if(nelements < 3) nelements = 3; /* First 3 slots reserved. If any higher + slots are used, preserve the contents. */ + + /* It seems the H5Z_FLAG_REVERSE flag doesn't work here, so we have to be + careful not to clobber any existing version info */ + if(values[0]==0) values[0] = H5PY_FILTER_LZF_VERSION; + if(values[1]==0) values[1] = LZF_VERSION; + + ndims = H5Pget_chunk(dcpl, 32, chunkdims); + if(ndims<0) return -1; + if(ndims>32){ + PUSH_ERR("lzf_set_local", H5E_CALLBACK, "Chunk rank exceeds limit"); + return -1; + } + + bufsize = H5Tget_size(type); + if(bufsize==0) return -1; + + for(i=0;i=3)&&(cd_values[2]!=0)){ + outbuf_size = cd_values[2]; /* Precomputed buffer guess */ + }else{ + outbuf_size = (*buf_size); + } + +#ifdef H5PY_LZF_DEBUG + fprintf(stderr, "Decompress %d chunk w/buffer %d\n", nbytes, outbuf_size); +#endif + + while(!status){ + + free(outbuf); + outbuf = malloc(outbuf_size); + + if(outbuf == NULL){ + PUSH_ERR("lzf_filter", H5E_CALLBACK, "Can't allocate decompression buffer"); + goto failed; + } + + status = lzf_decompress(*buf, nbytes, outbuf, outbuf_size); + + if(!status){ /* compression failed */ + + if(errno == E2BIG){ + outbuf_size += (*buf_size); +#ifdef H5PY_LZF_DEBUG + fprintf(stderr, " Too small: %d\n", outbuf_size); +#endif + } else if(errno == EINVAL) { + + PUSH_ERR("lzf_filter", H5E_CALLBACK, "Invalid data for LZF decompression"); + goto failed; + + } else { + PUSH_ERR("lzf_filter", H5E_CALLBACK, "Unknown LZF decompression error"); + goto failed; + } + + } /* if !status */ + + } /* while !status */ + + } /* compressing vs decompressing */ + + if(status != 0){ + + free(*buf); + *buf = outbuf; + *buf_size = outbuf_size; + + return status; /* Size of compressed/decompressed data */ + } + + failed: + + free(outbuf); + return 0; + +} /* End filter function */ + + + + + + + + + + + + + diff --git a/target/bitshuffle-0.3.4/lzf/lzf_filter.h b/target/bitshuffle-0.3.4/lzf/lzf_filter.h new file mode 100644 index 00000000..27dff83a --- /dev/null +++ b/target/bitshuffle-0.3.4/lzf/lzf_filter.h @@ -0,0 +1,38 @@ +/***** Preamble block ********************************************************* +* +* This file is part of h5py, a low-level Python interface to the HDF5 library. +* +* Copyright (C) 2008 Andrew Collette +* http://h5py.alfven.org +* License: BSD (See LICENSE.txt for full license) +* +* $Date$ +* +****** End preamble block ****************************************************/ + + +#ifndef H5PY_LZF_H +#define H5PY_LZF_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Filter revision number, starting at 1 */ +#define H5PY_FILTER_LZF_VERSION 4 + +/* Filter ID registered with the HDF Group as of 2/6/09. For maintenance + requests, contact the filter author directly. */ +#define H5PY_FILTER_LZF 32000 + +/* Register the filter with the library. Returns a negative value on failure, + and a non-negative value on success. +*/ +int register_lzf(void); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/target/bitshuffle-0.3.4/requirements.txt b/target/bitshuffle-0.3.4/requirements.txt new file mode 100644 index 00000000..2f0d0fbb --- /dev/null +++ b/target/bitshuffle-0.3.4/requirements.txt @@ -0,0 +1,5 @@ +# Order matters +setuptools>=0.7 +Cython>=0.19 +numpy>=1.6.1 +h5py>=2.4.0 --no-binary=h5py diff --git a/target/bitshuffle-0.3.4/setup.cfg.example b/target/bitshuffle-0.3.4/setup.cfg.example new file mode 100644 index 00000000..6bd2ccfb --- /dev/null +++ b/target/bitshuffle-0.3.4/setup.cfg.example @@ -0,0 +1,10 @@ +[install] +# These control the installation of the hdf5 dynamically loaded filter plugin. +h5plugin = 0 +h5plugin-dir = /usr/local/hdf5/lib/plugin + +[build_ext] +# Whether to compile with OpenMP multi-threading. Default is system dependant: +# False on OSX (since the clang compiler does not yet support OpenMP) and True +# otherwise. +omp = 1 diff --git a/target/bitshuffle-0.3.4/setup.py b/target/bitshuffle-0.3.4/setup.py new file mode 100644 index 00000000..5417dbde --- /dev/null +++ b/target/bitshuffle-0.3.4/setup.py @@ -0,0 +1,303 @@ +from __future__ import absolute_import, division, print_function +# I didn't import unicode_literals. They break setuptools or Cython in python +# 2.7, but python 3 seems to be happy with them. + +import glob +import os +from os import path +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as build_ext_ +from setuptools.command.develop import develop as develop_ +from setuptools.command.install import install as install_ +import shutil +import subprocess +import sys + + +VERSION_MAJOR = 0 +VERSION_MINOR = 3 +VERSION_POINT = 4 + +# Only unset in the 'release' branch and in tags. +VERSION_DEV = 0 + +VERSION = "%d.%d.%d" % (VERSION_MAJOR, VERSION_MINOR, VERSION_POINT) +if VERSION_DEV: + VERSION = VERSION + ".dev%d" % VERSION_DEV + + +COMPILE_FLAGS = ['-O3', '-ffast-math', '-march=native', '-std=c99'] +# Cython breaks strict aliasing rules. +COMPILE_FLAGS += ["-fno-strict-aliasing"] + +MACROS = [ + ('BSHUF_VERSION_MAJOR', VERSION_MAJOR), + ('BSHUF_VERSION_MINOR', VERSION_MINOR), + ('BSHUF_VERSION_POINT', VERSION_POINT), +] + + +H5PLUGINS_DEFAULT = '/usr/local/hdf5/lib/plugin' + +# OSX's clang compliler does not support OpenMP. +if sys.platform == 'darwin': + OMP_DEFAULT = False +else: + OMP_DEFAULT = True + +FALLBACK_CONFIG = { + 'include_dirs': [], + 'library_dirs': [], + 'libraries': [], + 'extra_compile_args': [], + 'extra_link_args': [], +} + +if sys.platform == 'darwin': + # putting here both macports and homebrew paths will generate + # "ld: warning: dir not found" at the linking phase + FALLBACK_CONFIG['include_dirs'] += ['/opt/local/include'] # macports + FALLBACK_CONFIG['library_dirs'] += ['/opt/local/lib'] # macports + FALLBACK_CONFIG['include_dirs'] += ['/usr/local/include'] # homebrew + FALLBACK_CONFIG['library_dirs'] += ['/usr/local/lib'] # homebrew +elif sys.platform.startswith('freebsd'): + FALLBACK_CONFIG['include_dirs'] += ['/usr/local/include'] # homebrew + FALLBACK_CONFIG['library_dirs'] += ['/usr/local/lib'] # homebrew + +FALLBACK_CONFIG['include_dirs'] = [d for d in FALLBACK_CONFIG['include_dirs'] + if path.isdir(d)] +FALLBACK_CONFIG['library_dirs'] = [d for d in FALLBACK_CONFIG['library_dirs'] + if path.isdir(d)] + + +def pkgconfig(*packages, **kw): + config = kw.setdefault('config', {}) + optional_args = kw.setdefault('optional', '') + flag_map = {'include_dirs': ['--cflags-only-I', 2], + 'library_dirs': ['--libs-only-L', 2], + 'libraries': ['--libs-only-l', 2], + 'extra_compile_args': ['--cflags-only-other', 0], + 'extra_link_args': ['--libs-only-other', 0], + } + for package in packages: + try: + subprocess.check_output(["pkg-config", package]) + except (subprocess.CalledProcessError, OSError): + print("Can't find %s with pkg-config fallback to " + "static config" % package) + for distutils_key in flag_map: + config.setdefault(distutils_key, []).extend( + FALLBACK_CONFIG[distutils_key]) + config['libraries'].append(package) + else: + for distutils_key, (pkg_option, n) in flag_map.items(): + items = subprocess.check_output( + ['pkg-config', optional_args, pkg_option, package] + ).decode('utf8').split() + opt = config.setdefault(distutils_key, []) + opt.extend([i[n:] for i in items]) + return config + + +ext_bshuf = Extension( + "bitshuffle.ext", + sources=["bitshuffle/ext.pyx", "src/bitshuffle.c", + "src/bitshuffle_core.c", "src/iochain.c", + "lz4/lz4.c"], + include_dirs=["src/", "lz4/"], + depends=["src/bitshuffle.h", "src/bitshuffle_core.h", + "src/iochain.h", "lz4/lz4.h"], + libraries=[], + extra_compile_args=COMPILE_FLAGS, + define_macros=MACROS, +) + +h5filter = Extension( + "bitshuffle.h5", + sources=["bitshuffle/h5.pyx", "src/bshuf_h5filter.c", + "src/bitshuffle.c", "src/bitshuffle_core.c", + "src/iochain.c", "lz4/lz4.c"], + depends=["src/bitshuffle.h", "src/bitshuffle_core.h", + "src/iochain.h", "src/bshuf_h5filter.h", + "lz4/lz4.h"], + define_macros=MACROS, + **pkgconfig("hdf5", config=dict( + include_dirs=["src/", "lz4/"], + extra_compile_args=COMPILE_FLAGS)) +) + +filter_plugin = Extension( + "bitshuffle.plugin.libh5bshuf", + sources=["src/bshuf_h5plugin.c", "src/bshuf_h5filter.c", + "src/bitshuffle.c", "src/bitshuffle_core.c", + "src/iochain.c", "lz4/lz4.c"], + depends=["src/bitshuffle.h", "src/bitshuffle_core.h", + "src/iochain.h", 'src/bshuf_h5filter.h', + "lz4/lz4.h"], + define_macros=MACROS, + **pkgconfig("hdf5", config=dict( + include_dirs=["src/", "lz4/"], + extra_compile_args=['-fPIC', '-g'] + COMPILE_FLAGS)) +) + +lzf_plugin = Extension( + "bitshuffle.plugin.libh5LZF", + sources=["src/lzf_h5plugin.c", "lzf/lzf_filter.c", + "lzf/lzf/lzf_c.c", "lzf/lzf/lzf_d.c"], + depends=["lzf/lzf_filter.h", "lzf/lzf/lzf.h", + "lzf/lzf/lzfP.h"], + **pkgconfig("hdf5", config=dict( + include_dirs=["lzf/", "lzf/lzf/"], + extra_compile_args=['-fPIC', '-g'] + COMPILE_FLAGS)) +) + + +EXTENSIONS = [ext_bshuf, h5filter] +# Check for plugin hdf5 plugin support (hdf5 >= 1.8.11) +HDF5_PLUGIN_SUPPORT = False +for p in ["/usr/include"] + pkgconfig("hdf5")["include_dirs"]: + if os.path.exists(os.path.join(p, "H5PLextern.h")): + HDF5_PLUGIN_SUPPORT = True + +if HDF5_PLUGIN_SUPPORT: + EXTENSIONS.extend([filter_plugin, lzf_plugin]) + + +class develop(develop_): + def run(self): + # Dummy directory for copying build plugins. + if not path.isdir('bitshuffle/plugin'): + os.mkdir('bitshuffle/plugin') + develop_.run(self) + + +# Custom installation to include installing dynamic filters. +class install(install_): + user_options = install_.user_options + [ + ('h5plugin', None, + 'Install HDF5 filter plugins for use outside of python.'), + ('h5plugin-dir=', None, + 'Where to install filter plugins. Default %s.' % H5PLUGINS_DEFAULT), + ] + + def initialize_options(self): + install_.initialize_options(self) + self.h5plugin = False + self.h5plugin_dir = H5PLUGINS_DEFAULT + + def finalize_options(self): + install_.finalize_options(self) + if self.h5plugin not in ('0', '1', True, False): + raise ValueError("Invalid h5plugin argument. Mut be '0' or '1'.") + self.h5plugin = int(self.h5plugin) + self.h5plugin_dir = path.abspath(self.h5plugin_dir) + + def run(self): + install_.run(self) + if self.h5plugin: + if not HDF5_PLUGIN_SUPPORT: + print("HDF5 < 1.8.11, not installing filter plugins.") + return + plugin_build = path.join(self.build_lib, "bitshuffle", "plugin") + try: + os.makedirs(self.h5plugin_dir) + except OSError as e: + if e.args[0] == 17: + # Directory already exists, this is fine. + pass + else: + raise + plugin_libs = glob.glob(path.join(plugin_build, "*")) + for plugin_lib in plugin_libs: + plugin_name = path.split(plugin_lib)[1] + shutil.copy2(plugin_lib, + path.join(self.h5plugin_dir, plugin_name)) + print("Installed HDF5 filter plugins to %s" % self.h5plugin_dir) + + +# Command line or site.cfg specification of OpenMP. +class build_ext(build_ext_): + user_options = build_ext_.user_options + [ + ('omp=', None, "Whether to compile with OpenMP threading. Default" + " on current system is %s." % str(OMP_DEFAULT)) + ] + boolean_options = build_ext_.boolean_options + ['omp'] + + def initialize_options(self): + build_ext_.initialize_options(self) + self.omp = OMP_DEFAULT + + def finalize_options(self): + # For some reason this gets run twice. Careful to print messages and + # add arguments only one time. + build_ext_.finalize_options(self) + + if self.omp not in ('0', '1', True, False): + raise ValueError("Invalid omp argument. Mut be '0' or '1'.") + self.omp = int(self.omp) + + import numpy as np + ext_bshuf.include_dirs.append(np.get_include()) + + if self.omp: + if not hasattr(self, "_printed_omp_message"): + self._printed_omp_message = True + print("\n#################################") + print("# Compiling with OpenMP support #") + print("#################################\n") + # More portable to pass -fopenmp to linker. + # self.libraries += ['gomp'] + for e in self.extensions: + if '-fopenmp' not in e.extra_compile_args: + e.extra_compile_args += ['-fopenmp'] + if '-fopenmp' not in e.extra_link_args: + e.extra_link_args += ['-fopenmp'] + + # Required only by old version of setuptools < 18.0 + from Cython.Build import cythonize + self.extensions = cythonize(self.extensions) + for ext in self.extensions: + ext._needs_stub = False + + +# Don't install numpy/cython/hdf5 if not needed +for cmd in ["sdist", "clean", + "--help", "--help-commands", "--version"]: + if cmd in sys.argv: + setup_requires = [] + break +else: + setup_requires = ["Cython>=0.19", "numpy>=1.6.1"] + +with open('requirements.txt') as f: + requires = f.read().splitlines() + requires = [r.split()[0] for r in requires] + +with open('README.rst') as r: + long_description = r.read() + +# TODO hdf5 support should be an "extra". Figure out how to set this up. +setup( + name='bitshuffle', + version=VERSION, + + packages=['bitshuffle', 'bitshuffle.tests'], + scripts=[], + ext_modules=EXTENSIONS, + cmdclass={'build_ext': build_ext, 'install': install, 'develop': develop}, + setup_requires=setup_requires, + install_requires=requires, + # extras_require={'H5': ["h5py"]}, + package_data={'': ['data/*']}, + + # metadata for upload to PyPI + author="Kiyoshi Wesley Masui", + author_email="kiyo@physics.ubc.ca", + description="Bitshuffle filter for improving typed data compression.", + long_description=long_description, + license="MIT", + url="https://github.com/kiyo-masui/bitshuffle", + download_url=("https://github.com/kiyo-masui/bitshuffle/tarball/%s" + % VERSION), + keywords=['compression', 'hdf5', 'numpy'], +) diff --git a/target/bitshuffle-0.3.4/src/bitshuffle.c b/target/bitshuffle-0.3.4/src/bitshuffle.c new file mode 100644 index 00000000..54ff045f --- /dev/null +++ b/target/bitshuffle-0.3.4/src/bitshuffle.c @@ -0,0 +1,165 @@ +/* + * Bitshuffle - Filter for improving compression of typed binary data. + * + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + */ + +#include "bitshuffle.h" +#include "bitshuffle_core.h" +#include "bitshuffle_internals.h" +#include "lz4.h" + +#include +#include + + +// Constants. +// Use fast decompression instead of safe decompression for LZ4. +#define BSHUF_LZ4_DECOMPRESS_FAST + + +// Macros. +#define CHECK_ERR_FREE_LZ(count, buf) if (count < 0) { \ + free(buf); return count - 1000; } + + +/* Bitshuffle and compress a single block. */ +int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr, \ + const size_t size, const size_t elem_size) { + + int64_t nbytes, count; + void *tmp_buf_bshuf; + void *tmp_buf_lz4; + size_t this_iter; + const void *in; + void *out; + + tmp_buf_bshuf = malloc(size * elem_size); + if (tmp_buf_bshuf == NULL) return -1; + + tmp_buf_lz4 = malloc(LZ4_compressBound(size * elem_size)); + if (tmp_buf_lz4 == NULL){ + free(tmp_buf_bshuf); + return -1; + } + + + in = ioc_get_in(C_ptr, &this_iter); + ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size)); + + count = bshuf_trans_bit_elem(in, tmp_buf_bshuf, size, elem_size); + if (count < 0) { + free(tmp_buf_lz4); + free(tmp_buf_bshuf); + return count; + } + nbytes = LZ4_compress((const char*) tmp_buf_bshuf, (char*) tmp_buf_lz4, size * elem_size); + free(tmp_buf_bshuf); + CHECK_ERR_FREE_LZ(nbytes, tmp_buf_lz4); + + out = ioc_get_out(C_ptr, &this_iter); + ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + nbytes + 4)); + + bshuf_write_uint32_BE(out, nbytes); + memcpy((char *) out + 4, tmp_buf_lz4, nbytes); + + free(tmp_buf_lz4); + + return nbytes + 4; +} + + +/* Decompress and bitunshuffle a single block. */ +int64_t bshuf_decompress_lz4_block(ioc_chain *C_ptr, + const size_t size, const size_t elem_size) { + + int64_t nbytes, count; + void *out, *tmp_buf; + const void *in; + size_t this_iter; + int32_t nbytes_from_header; + + in = ioc_get_in(C_ptr, &this_iter); + nbytes_from_header = bshuf_read_uint32_BE(in); + ioc_set_next_in(C_ptr, &this_iter, + (void*) ((char*) in + nbytes_from_header + 4)); + + out = ioc_get_out(C_ptr, &this_iter); + ioc_set_next_out(C_ptr, &this_iter, + (void *) ((char *) out + size * elem_size)); + + tmp_buf = malloc(size * elem_size); + if (tmp_buf == NULL) return -1; + +#ifdef BSHUF_LZ4_DECOMPRESS_FAST + nbytes = LZ4_decompress_fast((const char*) in + 4, (char*) tmp_buf, size * elem_size); + CHECK_ERR_FREE_LZ(nbytes, tmp_buf); + if (nbytes != nbytes_from_header) { + free(tmp_buf); + return -91; + } +#else + nbytes = LZ4_decompress_safe((const char*) in + 4, (char *) tmp_buf, nbytes_from_header, + size * elem_size); + CHECK_ERR_FREE_LZ(nbytes, tmp_buf); + if (nbytes != size * elem_size) { + free(tmp_buf); + return -91; + } + nbytes = nbytes_from_header; +#endif + count = bshuf_untrans_bit_elem(tmp_buf, out, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + nbytes += 4; + + free(tmp_buf); + return nbytes; +} + + +/* ---- Public functions ---- + * + * See header file for description and usage. + * + */ + +size_t bshuf_compress_lz4_bound(const size_t size, + const size_t elem_size, size_t block_size) { + + size_t bound, leftover; + + if (block_size == 0) { + block_size = bshuf_default_block_size(elem_size); + } + if (block_size % BSHUF_BLOCKED_MULT) return -81; + + // Note that each block gets a 4 byte header. + // Size of full blocks. + bound = (LZ4_compressBound(block_size * elem_size) + 4) * (size / block_size); + // Size of partial blocks, if any. + leftover = ((size % block_size) / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT; + if (leftover) bound += LZ4_compressBound(leftover * elem_size) + 4; + // Size of uncompressed data not fitting into any blocks. + bound += (size % BSHUF_BLOCKED_MULT) * elem_size; + return bound; +} + + +int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size, + const size_t elem_size, size_t block_size) { + return bshuf_blocked_wrap_fun(&bshuf_compress_lz4_block, in, out, size, + elem_size, block_size); +} + + +int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size, + const size_t elem_size, size_t block_size) { + return bshuf_blocked_wrap_fun(&bshuf_decompress_lz4_block, in, out, size, + elem_size, block_size); +} + diff --git a/target/bitshuffle-0.3.4/src/bitshuffle.h b/target/bitshuffle-0.3.4/src/bitshuffle.h new file mode 100644 index 00000000..3df95f47 --- /dev/null +++ b/target/bitshuffle-0.3.4/src/bitshuffle.h @@ -0,0 +1,123 @@ +/* + * Bitshuffle - Filter for improving compression of typed binary data. + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + * + * Header File + * + * Worker routines return an int64_t which is the number of bytes processed + * if positive or an error code if negative. + * + * Error codes: + * -1 : Failed to allocate memory. + * -11 : Missing SSE. + * -12 : Missing AVX. + * -80 : Input size not a multiple of 8. + * -81 : block_size not multiple of 8. + * -91 : Decompression error, wrong number of bytes processed. + * -1YYY : Error internal to compression routine with error code -YYY. + */ + + +#ifndef BITSHUFFLE_H +#define BITSHUFFLE_H + +#include +#include "bitshuffle_core.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* ---- bshuf_compress_lz4_bound ---- + * + * Bound on size of data compressed with *bshuf_compress_lz4*. + * + * Parameters + * ---------- + * size : number of elements in input + * elem_size : element size of typed data + * block_size : Process in blocks of this many elements. Pass 0 to + * select automatically (recommended). + * + * Returns + * ------- + * Bound on compressed data size. + * + */ +size_t bshuf_compress_lz4_bound(const size_t size, + const size_t elem_size, size_t block_size); + + +/* ---- bshuf_compress_lz4 ---- + * + * Bitshuffled and compress the data using LZ4. + * + * Transpose within elements, in blocks of data of *block_size* elements then + * compress the blocks using LZ4. In the output buffer, each block is prefixed + * by a 4 byte integer giving the compressed size of that block. + * + * Output buffer must be large enough to hold the compressed data. This could + * be in principle substantially larger than the input buffer. Use the routine + * *bshuf_compress_lz4_bound* to get an upper limit. + * + * Parameters + * ---------- + * in : input buffer, must be of size * elem_size bytes + * out : output buffer, must be large enough to hold data. + * size : number of elements in input + * elem_size : element size of typed data + * block_size : Process in blocks of this many elements. Pass 0 to + * select automatically (recommended). + * + * Returns + * ------- + * number of bytes used in output buffer, negative error-code if failed. + * + */ +int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size, const size_t + elem_size, size_t block_size); + + +/* ---- bshuf_decompress_lz4 ---- + * + * Undo compression and bitshuffling. + * + * Decompress data then un-bitshuffle it in blocks of *block_size* elements. + * + * To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size* + * must patch the parameters used to compress the data. + * + * NOT TO BE USED WITH UNTRUSTED DATA: This routine uses the function + * LZ4_decompress_fast from LZ4, which does not protect against maliciously + * formed datasets. By modifying the compressed data, this function could be + * coerced into leaving the boundaries of the input buffer. + * + * Parameters + * ---------- + * in : input buffer + * out : output buffer, must be of size * elem_size bytes + * size : number of elements in input + * elem_size : element size of typed data + * block_size : Process in blocks of this many elements. Pass 0 to + * select automatically (recommended). + * + * Returns + * ------- + * number of bytes consumed in *input* buffer, negative error-code if failed. + * + */ +int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size, + const size_t elem_size, size_t block_size); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // BITSHUFFLE_H diff --git a/target/bitshuffle-0.3.4/src/bitshuffle_core.c b/target/bitshuffle-0.3.4/src/bitshuffle_core.c new file mode 100644 index 00000000..583e4fe8 --- /dev/null +++ b/target/bitshuffle-0.3.4/src/bitshuffle_core.c @@ -0,0 +1,1333 @@ +/* + * Bitshuffle - Filter for improving compression of typed binary data. + * + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + */ + +#include "bitshuffle_core.h" +#include "bitshuffle_internals.h" + +#include +#include + + +#if defined(__AVX2__) && defined (__SSE2__) +#define USEAVX2 +#endif + +#if defined(__SSE2__) +#define USESSE2 +#endif + + +// Conditional includes for SSE2 and AVX2. +#ifdef USEAVX2 +#include +#elif defined USESSE2 +#include +#endif + + +// Macros. +#define CHECK_MULT_EIGHT(n) if (n % 8) return -80; +#define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) + + +/* ---- Functions indicating compile time instruction set. ---- */ + +int bshuf_using_SSE2(void) { +#ifdef USESSE2 + return 1; +#else + return 0; +#endif +} + + +int bshuf_using_AVX2(void) { +#ifdef USEAVX2 + return 1; +#else + return 0; +#endif +} + + +/* ---- Worker code not requiring special instruction sets. ---- + * + * The following code does not use any x86 specific vectorized instructions + * and should compile on any machine + * + */ + +/* Transpose 8x8 bit array packed into a single quadword *x*. + * *t* is workspace. */ +#define TRANS_BIT_8X8(x, t) { \ + t = (x ^ (x >> 7)) & 0x00AA00AA00AA00AALL; \ + x = x ^ t ^ (t << 7); \ + t = (x ^ (x >> 14)) & 0x0000CCCC0000CCCCLL; \ + x = x ^ t ^ (t << 14); \ + t = (x ^ (x >> 28)) & 0x00000000F0F0F0F0LL; \ + x = x ^ t ^ (t << 28); \ + } + +/* Transpose 8x8 bit array along the diagonal from upper right + to lower left */ +#define TRANS_BIT_8X8_BE(x, t) { \ + t = (x ^ (x >> 9)) & 0x0055005500550055LL; \ + x = x ^ t ^ (t << 9); \ + t = (x ^ (x >> 18)) & 0x0000333300003333LL; \ + x = x ^ t ^ (t << 18); \ + t = (x ^ (x >> 36)) & 0x000000000F0F0F0FLL; \ + x = x ^ t ^ (t << 36); \ + } + +/* Transpose of an array of arbitrarily typed elements. */ +#define TRANS_ELEM_TYPE(in, out, lda, ldb, type_t) { \ + size_t ii, jj, kk; \ + const type_t* in_type = (const type_t*) in; \ + type_t* out_type = (type_t*) out; \ + for(ii = 0; ii + 7 < lda; ii += 8) { \ + for(jj = 0; jj < ldb; jj++) { \ + for(kk = 0; kk < 8; kk++) { \ + out_type[jj*lda + ii + kk] = \ + in_type[ii*ldb + kk * ldb + jj]; \ + } \ + } \ + } \ + for(ii = lda - lda % 8; ii < lda; ii ++) { \ + for(jj = 0; jj < ldb; jj++) { \ + out_type[jj*lda + ii] = in_type[ii*ldb + jj]; \ + } \ + } \ + } + + +/* Memory copy with bshuf call signature. For testing and profiling. */ +int64_t bshuf_copy(const void* in, void* out, const size_t size, + const size_t elem_size) { + + const char* in_b = (const char*) in; + char* out_b = (char*) out; + + memcpy(out_b, in_b, size * elem_size); + return size * elem_size; +} + + +/* Transpose bytes within elements, starting partway through input. */ +int64_t bshuf_trans_byte_elem_remainder(const void* in, void* out, const size_t size, + const size_t elem_size, const size_t start) { + + size_t ii, jj, kk; + const char* in_b = (const char*) in; + char* out_b = (char*) out; + + CHECK_MULT_EIGHT(start); + + if (size > start) { + // ii loop separated into 2 loops so the compiler can unroll + // the inner one. + for (ii = start; ii + 7 < size; ii += 8) { + for (jj = 0; jj < elem_size; jj++) { + for (kk = 0; kk < 8; kk++) { + out_b[jj * size + ii + kk] + = in_b[ii * elem_size + kk * elem_size + jj]; + } + } + } + for (ii = size - size % 8; ii < size; ii ++) { + for (jj = 0; jj < elem_size; jj++) { + out_b[jj * size + ii] = in_b[ii * elem_size + jj]; + } + } + } + return size * elem_size; +} + + +/* Transpose bytes within elements. */ +int64_t bshuf_trans_byte_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size) { + + return bshuf_trans_byte_elem_remainder(in, out, size, elem_size, 0); +} + + +/* Transpose bits within bytes. */ +int64_t bshuf_trans_bit_byte_remainder(const void* in, void* out, const size_t size, + const size_t elem_size, const size_t start_byte) { + + const uint64_t* in_b = (const uint64_t*) in; + uint8_t* out_b = (uint8_t*) out; + + uint64_t x, t; + + size_t ii, kk; + size_t nbyte = elem_size * size; + size_t nbyte_bitrow = nbyte / 8; + + uint64_t e=1; + const int little_endian = *(uint8_t *) &e == 1; + const size_t bit_row_skip = little_endian ? nbyte_bitrow : -nbyte_bitrow; + const int64_t bit_row_offset = little_endian ? 0 : 7 * nbyte_bitrow; + + CHECK_MULT_EIGHT(nbyte); + CHECK_MULT_EIGHT(start_byte); + + for (ii = start_byte / 8; ii < nbyte_bitrow; ii ++) { + x = in_b[ii]; + if (little_endian) { + TRANS_BIT_8X8(x, t); + } else { + TRANS_BIT_8X8_BE(x, t); + } + for (kk = 0; kk < 8; kk ++) { + out_b[bit_row_offset + kk * bit_row_skip + ii] = x; + x = x >> 8; + } + } + return size * elem_size; +} + + +/* Transpose bits within bytes. */ +int64_t bshuf_trans_bit_byte_scal(const void* in, void* out, const size_t size, + const size_t elem_size) { + + return bshuf_trans_bit_byte_remainder(in, out, size, elem_size, 0); +} + + +/* General transpose of an array, optimized for large element sizes. */ +int64_t bshuf_trans_elem(const void* in, void* out, const size_t lda, + const size_t ldb, const size_t elem_size) { + + size_t ii, jj; + const char* in_b = (const char*) in; + char* out_b = (char*) out; + for(ii = 0; ii < lda; ii++) { + for(jj = 0; jj < ldb; jj++) { + memcpy(&out_b[(jj*lda + ii) * elem_size], + &in_b[(ii*ldb + jj) * elem_size], elem_size); + } + } + return lda * ldb * elem_size; +} + + +/* Transpose rows of shuffled bits (size / 8 bytes) within groups of 8. */ +int64_t bshuf_trans_bitrow_eight(const void* in, void* out, const size_t size, + const size_t elem_size) { + + size_t nbyte_bitrow = size / 8; + + CHECK_MULT_EIGHT(size); + + return bshuf_trans_elem(in, out, 8, elem_size, nbyte_bitrow); +} + + +/* Transpose bits within elements. */ +int64_t bshuf_trans_bit_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; + void *tmp_buf; + + CHECK_MULT_EIGHT(size); + + tmp_buf = malloc(size * elem_size); + if (tmp_buf == NULL) return -1; + + count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_trans_bit_byte_scal(out, tmp_buf, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); + + free(tmp_buf); + + return count; +} + + +/* For data organized into a row for each bit (8 * elem_size rows), transpose + * the bytes. */ +int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t size, + const size_t elem_size) { + size_t ii, jj, kk, nbyte_row; + const char *in_b; + char *out_b; + + + in_b = (const char*) in; + out_b = (char*) out; + + nbyte_row = size / 8; + + CHECK_MULT_EIGHT(size); + + for (jj = 0; jj < elem_size; jj++) { + for (ii = 0; ii < nbyte_row; ii++) { + for (kk = 0; kk < 8; kk++) { + out_b[ii * 8 * elem_size + jj * 8 + kk] = \ + in_b[(jj * 8 + kk) * nbyte_row + ii]; + } + } + } + return size * elem_size; +} + + +/* Shuffle bits within the bytes of eight element blocks. */ +int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, \ + const size_t size, const size_t elem_size) { + + const char *in_b; + char *out_b; + uint64_t x, t; + size_t ii, jj, kk; + size_t nbyte, out_index; + + uint64_t e=1; + const int little_endian = *(uint8_t *) &e == 1; + const size_t elem_skip = little_endian ? elem_size : -elem_size; + const uint64_t elem_offset = little_endian ? 0 : 7 * elem_size; + + CHECK_MULT_EIGHT(size); + + in_b = (const char*) in; + out_b = (char*) out; + + nbyte = elem_size * size; + + for (jj = 0; jj < 8 * elem_size; jj += 8) { + for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { + x = *((uint64_t*) &in_b[ii + jj]); + if (little_endian) { + TRANS_BIT_8X8(x, t); + } else { + TRANS_BIT_8X8_BE(x, t); + } + for (kk = 0; kk < 8; kk++) { + out_index = ii + jj / 8 + elem_offset + kk * elem_skip; + *((uint8_t*) &out_b[out_index]) = x; + x = x >> 8; + } + } + } + return size * elem_size; +} + + +/* Untranspose bits within elements. */ +int64_t bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; + void *tmp_buf; + + CHECK_MULT_EIGHT(size); + + tmp_buf = malloc(size * elem_size); + if (tmp_buf == NULL) return -1; + + count = bshuf_trans_byte_bitrow_scal(in, tmp_buf, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_shuffle_bit_eightelem_scal(tmp_buf, out, size, elem_size); + + free(tmp_buf); + + return count; +} + + +/* ---- Worker code that uses SSE2 ---- + * + * The following code makes use of the SSE2 instruction set and specialized + * 16 byte registers. The SSE2 instructions are present on modern x86 + * processors. The first Intel processor microarchitecture supporting SSE2 was + * Pentium 4 (2000). + * + */ + +#ifdef USESSE2 + +/* Transpose bytes within elements for 16 bit elements. */ +int64_t bshuf_trans_byte_elem_SSE_16(const void* in, void* out, const size_t size) { + + size_t ii; + const char *in_b = (const char*) in; + char *out_b = (char*) out; + __m128i a0, b0, a1, b1; + + for (ii=0; ii + 15 < size; ii += 16) { + a0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 0*16]); + b0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 1*16]); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + + a0 = _mm_unpacklo_epi8(a1, b1); + b0 = _mm_unpackhi_epi8(a1, b1); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + + a0 = _mm_unpacklo_epi8(a1, b1); + b0 = _mm_unpackhi_epi8(a1, b1); + + _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); + _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); + } + return bshuf_trans_byte_elem_remainder(in, out, size, 2, + size - size % 16); +} + + +/* Transpose bytes within elements for 32 bit elements. */ +int64_t bshuf_trans_byte_elem_SSE_32(const void* in, void* out, const size_t size) { + + size_t ii; + const char *in_b; + char *out_b; + in_b = (const char*) in; + out_b = (char*) out; + __m128i a0, b0, c0, d0, a1, b1, c1, d1; + + for (ii=0; ii + 15 < size; ii += 16) { + a0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 0*16]); + b0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 1*16]); + c0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 2*16]); + d0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 3*16]); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + c1 = _mm_unpacklo_epi8(c0, d0); + d1 = _mm_unpackhi_epi8(c0, d0); + + a0 = _mm_unpacklo_epi8(a1, b1); + b0 = _mm_unpackhi_epi8(a1, b1); + c0 = _mm_unpacklo_epi8(c1, d1); + d0 = _mm_unpackhi_epi8(c1, d1); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + c1 = _mm_unpacklo_epi8(c0, d0); + d1 = _mm_unpackhi_epi8(c0, d0); + + a0 = _mm_unpacklo_epi64(a1, c1); + b0 = _mm_unpackhi_epi64(a1, c1); + c0 = _mm_unpacklo_epi64(b1, d1); + d0 = _mm_unpackhi_epi64(b1, d1); + + _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); + _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); + _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0); + _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0); + } + return bshuf_trans_byte_elem_remainder(in, out, size, 4, + size - size % 16); +} + + +/* Transpose bytes within elements for 64 bit elements. */ +int64_t bshuf_trans_byte_elem_SSE_64(const void* in, void* out, const size_t size) { + + size_t ii; + const char* in_b = (const char*) in; + char* out_b = (char*) out; + __m128i a0, b0, c0, d0, e0, f0, g0, h0; + __m128i a1, b1, c1, d1, e1, f1, g1, h1; + + for (ii=0; ii + 15 < size; ii += 16) { + a0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 0*16]); + b0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 1*16]); + c0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 2*16]); + d0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 3*16]); + e0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 4*16]); + f0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 5*16]); + g0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 6*16]); + h0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 7*16]); + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpackhi_epi8(a0, b0); + c1 = _mm_unpacklo_epi8(c0, d0); + d1 = _mm_unpackhi_epi8(c0, d0); + e1 = _mm_unpacklo_epi8(e0, f0); + f1 = _mm_unpackhi_epi8(e0, f0); + g1 = _mm_unpacklo_epi8(g0, h0); + h1 = _mm_unpackhi_epi8(g0, h0); + + a0 = _mm_unpacklo_epi8(a1, b1); + b0 = _mm_unpackhi_epi8(a1, b1); + c0 = _mm_unpacklo_epi8(c1, d1); + d0 = _mm_unpackhi_epi8(c1, d1); + e0 = _mm_unpacklo_epi8(e1, f1); + f0 = _mm_unpackhi_epi8(e1, f1); + g0 = _mm_unpacklo_epi8(g1, h1); + h0 = _mm_unpackhi_epi8(g1, h1); + + a1 = _mm_unpacklo_epi32(a0, c0); + b1 = _mm_unpackhi_epi32(a0, c0); + c1 = _mm_unpacklo_epi32(b0, d0); + d1 = _mm_unpackhi_epi32(b0, d0); + e1 = _mm_unpacklo_epi32(e0, g0); + f1 = _mm_unpackhi_epi32(e0, g0); + g1 = _mm_unpacklo_epi32(f0, h0); + h1 = _mm_unpackhi_epi32(f0, h0); + + a0 = _mm_unpacklo_epi64(a1, e1); + b0 = _mm_unpackhi_epi64(a1, e1); + c0 = _mm_unpacklo_epi64(b1, f1); + d0 = _mm_unpackhi_epi64(b1, f1); + e0 = _mm_unpacklo_epi64(c1, g1); + f0 = _mm_unpackhi_epi64(c1, g1); + g0 = _mm_unpacklo_epi64(d1, h1); + h0 = _mm_unpackhi_epi64(d1, h1); + + _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); + _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); + _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0); + _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0); + _mm_storeu_si128((__m128i *) &out_b[4*size + ii], e0); + _mm_storeu_si128((__m128i *) &out_b[5*size + ii], f0); + _mm_storeu_si128((__m128i *) &out_b[6*size + ii], g0); + _mm_storeu_si128((__m128i *) &out_b[7*size + ii], h0); + } + return bshuf_trans_byte_elem_remainder(in, out, size, 8, + size - size % 16); +} + + +/* Transpose bytes within elements using best SSE algorithm available. */ +int64_t bshuf_trans_byte_elem_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; + + // Trivial cases: power of 2 bytes. + switch (elem_size) { + case 1: + count = bshuf_copy(in, out, size, elem_size); + return count; + case 2: + count = bshuf_trans_byte_elem_SSE_16(in, out, size); + return count; + case 4: + count = bshuf_trans_byte_elem_SSE_32(in, out, size); + return count; + case 8: + count = bshuf_trans_byte_elem_SSE_64(in, out, size); + return count; + } + + // Worst case: odd number of bytes. Turns out that this is faster for + // (odd * 2) byte elements as well (hence % 4). + if (elem_size % 4) { + count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); + return count; + } + + // Multiple of power of 2: transpose hierarchically. + { + size_t nchunk_elem; + void* tmp_buf = malloc(size * elem_size); + if (tmp_buf == NULL) return -1; + + if ((elem_size % 8) == 0) { + nchunk_elem = elem_size / 8; + TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int64_t); + count = bshuf_trans_byte_elem_SSE_64(out, tmp_buf, + size * nchunk_elem); + bshuf_trans_elem(tmp_buf, out, 8, nchunk_elem, size); + } else if ((elem_size % 4) == 0) { + nchunk_elem = elem_size / 4; + TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int32_t); + count = bshuf_trans_byte_elem_SSE_32(out, tmp_buf, + size * nchunk_elem); + bshuf_trans_elem(tmp_buf, out, 4, nchunk_elem, size); + } else { + // Not used since scalar algorithm is faster. + nchunk_elem = elem_size / 2; + TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int16_t); + count = bshuf_trans_byte_elem_SSE_16(out, tmp_buf, + size * nchunk_elem); + bshuf_trans_elem(tmp_buf, out, 2, nchunk_elem, size); + } + + free(tmp_buf); + return count; + } +} + + +/* Transpose bits within bytes. */ +int64_t bshuf_trans_bit_byte_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + + size_t ii, kk; + const char* in_b = (const char*) in; + char* out_b = (char*) out; + uint16_t* out_ui16; + + int64_t count; + + size_t nbyte = elem_size * size; + + CHECK_MULT_EIGHT(nbyte); + + __m128i xmm; + int32_t bt; + + for (ii = 0; ii + 15 < nbyte; ii += 16) { + xmm = _mm_loadu_si128((__m128i *) &in_b[ii]); + for (kk = 0; kk < 8; kk++) { + bt = _mm_movemask_epi8(xmm); + xmm = _mm_slli_epi16(xmm, 1); + out_ui16 = (uint16_t*) &out_b[((7 - kk) * nbyte + ii) / 8]; + *out_ui16 = bt; + } + } + count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, + nbyte - nbyte % 16); + return count; +} + + +/* Transpose bits within elements. */ +int64_t bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + void* tmp_buf = malloc(size * elem_size); + if (tmp_buf == NULL) return -1; + + count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_trans_bit_byte_SSE(out, tmp_buf, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); + + free(tmp_buf); + + return count; +} + + +/* For data organized into a row for each bit (8 * elem_size rows), transpose + * the bytes. */ +int64_t bshuf_trans_byte_bitrow_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + + size_t ii, jj; + const char* in_b = (const char*) in; + char* out_b = (char*) out; + + CHECK_MULT_EIGHT(size); + + size_t nrows = 8 * elem_size; + size_t nbyte_row = size / 8; + + __m128i a0, b0, c0, d0, e0, f0, g0, h0; + __m128i a1, b1, c1, d1, e1, f1, g1, h1; + __m128 *as, *bs, *cs, *ds, *es, *fs, *gs, *hs; + + for (ii = 0; ii + 7 < nrows; ii += 8) { + for (jj = 0; jj + 15 < nbyte_row; jj += 16) { + a0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 0)*nbyte_row + jj]); + b0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 1)*nbyte_row + jj]); + c0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 2)*nbyte_row + jj]); + d0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 3)*nbyte_row + jj]); + e0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 4)*nbyte_row + jj]); + f0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 5)*nbyte_row + jj]); + g0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 6)*nbyte_row + jj]); + h0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 7)*nbyte_row + jj]); + + + a1 = _mm_unpacklo_epi8(a0, b0); + b1 = _mm_unpacklo_epi8(c0, d0); + c1 = _mm_unpacklo_epi8(e0, f0); + d1 = _mm_unpacklo_epi8(g0, h0); + e1 = _mm_unpackhi_epi8(a0, b0); + f1 = _mm_unpackhi_epi8(c0, d0); + g1 = _mm_unpackhi_epi8(e0, f0); + h1 = _mm_unpackhi_epi8(g0, h0); + + + a0 = _mm_unpacklo_epi16(a1, b1); + b0 = _mm_unpacklo_epi16(c1, d1); + c0 = _mm_unpackhi_epi16(a1, b1); + d0 = _mm_unpackhi_epi16(c1, d1); + + e0 = _mm_unpacklo_epi16(e1, f1); + f0 = _mm_unpacklo_epi16(g1, h1); + g0 = _mm_unpackhi_epi16(e1, f1); + h0 = _mm_unpackhi_epi16(g1, h1); + + + a1 = _mm_unpacklo_epi32(a0, b0); + b1 = _mm_unpackhi_epi32(a0, b0); + + c1 = _mm_unpacklo_epi32(c0, d0); + d1 = _mm_unpackhi_epi32(c0, d0); + + e1 = _mm_unpacklo_epi32(e0, f0); + f1 = _mm_unpackhi_epi32(e0, f0); + + g1 = _mm_unpacklo_epi32(g0, h0); + h1 = _mm_unpackhi_epi32(g0, h0); + + // We don't have a storeh instruction for integers, so interpret + // as a float. Have a storel (_mm_storel_epi64). + as = (__m128 *) &a1; + bs = (__m128 *) &b1; + cs = (__m128 *) &c1; + ds = (__m128 *) &d1; + es = (__m128 *) &e1; + fs = (__m128 *) &f1; + gs = (__m128 *) &g1; + hs = (__m128 *) &h1; + + _mm_storel_pi((__m64 *) &out_b[(jj + 0) * nrows + ii], *as); + _mm_storel_pi((__m64 *) &out_b[(jj + 2) * nrows + ii], *bs); + _mm_storel_pi((__m64 *) &out_b[(jj + 4) * nrows + ii], *cs); + _mm_storel_pi((__m64 *) &out_b[(jj + 6) * nrows + ii], *ds); + _mm_storel_pi((__m64 *) &out_b[(jj + 8) * nrows + ii], *es); + _mm_storel_pi((__m64 *) &out_b[(jj + 10) * nrows + ii], *fs); + _mm_storel_pi((__m64 *) &out_b[(jj + 12) * nrows + ii], *gs); + _mm_storel_pi((__m64 *) &out_b[(jj + 14) * nrows + ii], *hs); + + _mm_storeh_pi((__m64 *) &out_b[(jj + 1) * nrows + ii], *as); + _mm_storeh_pi((__m64 *) &out_b[(jj + 3) * nrows + ii], *bs); + _mm_storeh_pi((__m64 *) &out_b[(jj + 5) * nrows + ii], *cs); + _mm_storeh_pi((__m64 *) &out_b[(jj + 7) * nrows + ii], *ds); + _mm_storeh_pi((__m64 *) &out_b[(jj + 9) * nrows + ii], *es); + _mm_storeh_pi((__m64 *) &out_b[(jj + 11) * nrows + ii], *fs); + _mm_storeh_pi((__m64 *) &out_b[(jj + 13) * nrows + ii], *gs); + _mm_storeh_pi((__m64 *) &out_b[(jj + 15) * nrows + ii], *hs); + } + for (jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj ++) { + out_b[jj * nrows + ii + 0] = in_b[(ii + 0)*nbyte_row + jj]; + out_b[jj * nrows + ii + 1] = in_b[(ii + 1)*nbyte_row + jj]; + out_b[jj * nrows + ii + 2] = in_b[(ii + 2)*nbyte_row + jj]; + out_b[jj * nrows + ii + 3] = in_b[(ii + 3)*nbyte_row + jj]; + out_b[jj * nrows + ii + 4] = in_b[(ii + 4)*nbyte_row + jj]; + out_b[jj * nrows + ii + 5] = in_b[(ii + 5)*nbyte_row + jj]; + out_b[jj * nrows + ii + 6] = in_b[(ii + 6)*nbyte_row + jj]; + out_b[jj * nrows + ii + 7] = in_b[(ii + 7)*nbyte_row + jj]; + } + } + return size * elem_size; +} + + +/* Shuffle bits within the bytes of eight element blocks. */ +int64_t bshuf_shuffle_bit_eightelem_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + + CHECK_MULT_EIGHT(size); + + // With a bit of care, this could be written such that such that it is + // in_buf = out_buf safe. + const char* in_b = (const char*) in; + uint16_t* out_ui16 = (uint16_t*) out; + + size_t ii, jj, kk; + size_t nbyte = elem_size * size; + + __m128i xmm; + int32_t bt; + + if (elem_size % 2) { + bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size); + } else { + for (ii = 0; ii + 8 * elem_size - 1 < nbyte; + ii += 8 * elem_size) { + for (jj = 0; jj + 15 < 8 * elem_size; jj += 16) { + xmm = _mm_loadu_si128((__m128i *) &in_b[ii + jj]); + for (kk = 0; kk < 8; kk++) { + bt = _mm_movemask_epi8(xmm); + xmm = _mm_slli_epi16(xmm, 1); + size_t ind = (ii + jj / 8 + (7 - kk) * elem_size); + out_ui16[ind / 2] = bt; + } + } + } + } + return size * elem_size; +} + + +/* Untranspose bits within elements. */ +int64_t bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + void* tmp_buf = malloc(size * elem_size); + if (tmp_buf == NULL) return -1; + + count = bshuf_trans_byte_bitrow_SSE(in, tmp_buf, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_shuffle_bit_eightelem_SSE(tmp_buf, out, size, elem_size); + + free(tmp_buf); + + return count; +} + +#else // #ifdef USESSE2 + + +int64_t bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -11; +} + + +int64_t bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -11; +} + + +int64_t bshuf_trans_byte_bitrow_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -11; +} + + +int64_t bshuf_trans_bit_byte_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -11; +} + + +int64_t bshuf_trans_byte_elem_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -11; +} + + +int64_t bshuf_trans_byte_elem_SSE_64(const void* in, void* out, const size_t size) { + return -11; +} + + +int64_t bshuf_trans_byte_elem_SSE_32(const void* in, void* out, const size_t size) { + return -11; +} + + +int64_t bshuf_trans_byte_elem_SSE_16(const void* in, void* out, const size_t size) { + return -11; +} + + +int64_t bshuf_shuffle_bit_eightelem_SSE(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -11; +} + + +#endif // #ifdef USESSE2 + + +/* ---- Code that requires AVX2. Intel Haswell (2013) and later. ---- */ + +/* ---- Worker code that uses AVX2 ---- + * + * The following code makes use of the AVX2 instruction set and specialized + * 32 byte registers. The AVX2 instructions are present on newer x86 + * processors. The first Intel processor microarchitecture supporting AVX2 was + * Haswell (2013). + * + */ + +#ifdef USEAVX2 + +/* Transpose bits within bytes. */ +int64_t bshuf_trans_bit_byte_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + + size_t ii, kk; + const char* in_b = (const char*) in; + char* out_b = (char*) out; + int32_t* out_i32; + + size_t nbyte = elem_size * size; + + int64_t count; + + __m256i ymm; + int32_t bt; + + for (ii = 0; ii + 31 < nbyte; ii += 32) { + ymm = _mm256_loadu_si256((__m256i *) &in_b[ii]); + for (kk = 0; kk < 8; kk++) { + bt = _mm256_movemask_epi8(ymm); + ymm = _mm256_slli_epi16(ymm, 1); + out_i32 = (int32_t*) &out_b[((7 - kk) * nbyte + ii) / 8]; + *out_i32 = bt; + } + } + count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, + nbyte - nbyte % 32); + return count; +} + + +/* Transpose bits within elements. */ +int64_t bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + void* tmp_buf = malloc(size * elem_size); + if (tmp_buf == NULL) return -1; + + count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_trans_bit_byte_AVX(out, tmp_buf, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); + + free(tmp_buf); + + return count; +} + + +/* For data organized into a row for each bit (8 * elem_size rows), transpose + * the bytes. */ +int64_t bshuf_trans_byte_bitrow_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + + size_t hh, ii, jj, kk, mm; + const char* in_b = (const char*) in; + char* out_b = (char*) out; + + CHECK_MULT_EIGHT(size); + + size_t nrows = 8 * elem_size; + size_t nbyte_row = size / 8; + + if (elem_size % 4) return bshuf_trans_byte_bitrow_SSE(in, out, size, + elem_size); + + __m256i ymm_0[8]; + __m256i ymm_1[8]; + __m256i ymm_storeage[8][4]; + + for (jj = 0; jj + 31 < nbyte_row; jj += 32) { + for (ii = 0; ii + 3 < elem_size; ii += 4) { + for (hh = 0; hh < 4; hh ++) { + + for (kk = 0; kk < 8; kk ++){ + ymm_0[kk] = _mm256_loadu_si256((__m256i *) &in_b[ + (ii * 8 + hh * 8 + kk) * nbyte_row + jj]); + } + + for (kk = 0; kk < 4; kk ++){ + ymm_1[kk] = _mm256_unpacklo_epi8(ymm_0[kk * 2], + ymm_0[kk * 2 + 1]); + ymm_1[kk + 4] = _mm256_unpackhi_epi8(ymm_0[kk * 2], + ymm_0[kk * 2 + 1]); + } + + for (kk = 0; kk < 2; kk ++){ + for (mm = 0; mm < 2; mm ++){ + ymm_0[kk * 4 + mm] = _mm256_unpacklo_epi16( + ymm_1[kk * 4 + mm * 2], + ymm_1[kk * 4 + mm * 2 + 1]); + ymm_0[kk * 4 + mm + 2] = _mm256_unpackhi_epi16( + ymm_1[kk * 4 + mm * 2], + ymm_1[kk * 4 + mm * 2 + 1]); + } + } + + for (kk = 0; kk < 4; kk ++){ + ymm_1[kk * 2] = _mm256_unpacklo_epi32(ymm_0[kk * 2], + ymm_0[kk * 2 + 1]); + ymm_1[kk * 2 + 1] = _mm256_unpackhi_epi32(ymm_0[kk * 2], + ymm_0[kk * 2 + 1]); + } + + for (kk = 0; kk < 8; kk ++){ + ymm_storeage[kk][hh] = ymm_1[kk]; + } + } + + for (mm = 0; mm < 8; mm ++) { + + for (kk = 0; kk < 4; kk ++){ + ymm_0[kk] = ymm_storeage[mm][kk]; + } + + ymm_1[0] = _mm256_unpacklo_epi64(ymm_0[0], ymm_0[1]); + ymm_1[1] = _mm256_unpacklo_epi64(ymm_0[2], ymm_0[3]); + ymm_1[2] = _mm256_unpackhi_epi64(ymm_0[0], ymm_0[1]); + ymm_1[3] = _mm256_unpackhi_epi64(ymm_0[2], ymm_0[3]); + + ymm_0[0] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 32); + ymm_0[1] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 32); + ymm_0[2] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 49); + ymm_0[3] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 49); + + _mm256_storeu_si256((__m256i *) &out_b[ + (jj + mm * 2 + 0 * 16) * nrows + ii * 8], ymm_0[0]); + _mm256_storeu_si256((__m256i *) &out_b[ + (jj + mm * 2 + 0 * 16 + 1) * nrows + ii * 8], ymm_0[1]); + _mm256_storeu_si256((__m256i *) &out_b[ + (jj + mm * 2 + 1 * 16) * nrows + ii * 8], ymm_0[2]); + _mm256_storeu_si256((__m256i *) &out_b[ + (jj + mm * 2 + 1 * 16 + 1) * nrows + ii * 8], ymm_0[3]); + } + } + } + for (ii = 0; ii < nrows; ii ++ ) { + for (jj = nbyte_row - nbyte_row % 32; jj < nbyte_row; jj ++) { + out_b[jj * nrows + ii] = in_b[ii * nbyte_row + jj]; + } + } + return size * elem_size; +} + + +/* Shuffle bits within the bytes of eight element blocks. */ +int64_t bshuf_shuffle_bit_eightelem_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + + CHECK_MULT_EIGHT(size); + + // With a bit of care, this could be written such that such that it is + // in_buf = out_buf safe. + const char* in_b = (const char*) in; + char* out_b = (char*) out; + + size_t ii, jj, kk; + size_t nbyte = elem_size * size; + + __m256i ymm; + int32_t bt; + + if (elem_size % 4) { + return bshuf_shuffle_bit_eightelem_SSE(in, out, size, elem_size); + } else { + for (jj = 0; jj + 31 < 8 * elem_size; jj += 32) { + for (ii = 0; ii + 8 * elem_size - 1 < nbyte; + ii += 8 * elem_size) { + ymm = _mm256_loadu_si256((__m256i *) &in_b[ii + jj]); + for (kk = 0; kk < 8; kk++) { + bt = _mm256_movemask_epi8(ymm); + ymm = _mm256_slli_epi16(ymm, 1); + size_t ind = (ii + jj / 8 + (7 - kk) * elem_size); + * (int32_t *) &out_b[ind] = bt; + } + } + } + } + return size * elem_size; +} + + +/* Untranspose bits within elements. */ +int64_t bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; + + CHECK_MULT_EIGHT(size); + + void* tmp_buf = malloc(size * elem_size); + if (tmp_buf == NULL) return -1; + + count = bshuf_trans_byte_bitrow_AVX(in, tmp_buf, size, elem_size); + CHECK_ERR_FREE(count, tmp_buf); + count = bshuf_shuffle_bit_eightelem_AVX(tmp_buf, out, size, elem_size); + + free(tmp_buf); + return count; +} + + +#else // #ifdef USEAVX2 + +int64_t bshuf_trans_bit_byte_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -12; +} + + +int64_t bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -12; +} + + +int64_t bshuf_trans_byte_bitrow_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -12; +} + + +int64_t bshuf_shuffle_bit_eightelem_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -12; +} + + +int64_t bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size, + const size_t elem_size) { + return -12; +} + +#endif // #ifdef USEAVX2 + + +/* ---- Drivers selecting best instruction set at compile time. ---- */ + +int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; +#ifdef USEAVX2 + count = bshuf_trans_bit_elem_AVX(in, out, size, elem_size); +#elif defined(USESSE2) + count = bshuf_trans_bit_elem_SSE(in, out, size, elem_size); +#else + count = bshuf_trans_bit_elem_scal(in, out, size, elem_size); +#endif + return count; +} + + +int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size, + const size_t elem_size) { + + int64_t count; +#ifdef USEAVX2 + count = bshuf_untrans_bit_elem_AVX(in, out, size, elem_size); +#elif defined(USESSE2) + count = bshuf_untrans_bit_elem_SSE(in, out, size, elem_size); +#else + count = bshuf_untrans_bit_elem_scal(in, out, size, elem_size); +#endif + return count; +} + + +/* ---- Wrappers for implementing blocking ---- */ + +/* Wrap a function for processing a single block to process an entire buffer in + * parallel. */ +int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out, \ + const size_t size, const size_t elem_size, size_t block_size) { + + size_t ii; + int64_t err = 0; + int64_t count, cum_count=0; + size_t last_block_size; + size_t leftover_bytes; + size_t this_iter; + char *last_in; + char *last_out; + + + ioc_chain C; + ioc_init(&C, in, out); + + + if (block_size == 0) { + block_size = bshuf_default_block_size(elem_size); + } + if (block_size % BSHUF_BLOCKED_MULT) return -81; + +#if defined(_OPENMP) + #pragma omp parallel for schedule(dynamic, 1) \ + private(count) reduction(+ : cum_count) +#endif + for (ii = 0; ii < size / block_size; ii ++) { + count = fun(&C, block_size, elem_size); + if (count < 0) err = count; + cum_count += count; + } + + last_block_size = size % block_size; + last_block_size = last_block_size - last_block_size % BSHUF_BLOCKED_MULT; + if (last_block_size) { + count = fun(&C, last_block_size, elem_size); + if (count < 0) err = count; + cum_count += count; + } + + if (err < 0) return err; + + leftover_bytes = size % BSHUF_BLOCKED_MULT * elem_size; + //this_iter; + last_in = (char *) ioc_get_in(&C, &this_iter); + ioc_set_next_in(&C, &this_iter, (void *) (last_in + leftover_bytes)); + last_out = (char *) ioc_get_out(&C, &this_iter); + ioc_set_next_out(&C, &this_iter, (void *) (last_out + leftover_bytes)); + + memcpy(last_out, last_in, leftover_bytes); + + ioc_destroy(&C); + + return cum_count + leftover_bytes; +} + + +/* Bitshuffle a single block. */ +int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \ + const size_t size, const size_t elem_size) { + + size_t this_iter; + const void *in; + void *out; + int64_t count; + + + + in = ioc_get_in(C_ptr, &this_iter); + ioc_set_next_in(C_ptr, &this_iter, + (void*) ((char*) in + size * elem_size)); + out = ioc_get_out(C_ptr, &this_iter); + ioc_set_next_out(C_ptr, &this_iter, + (void *) ((char *) out + size * elem_size)); + + count = bshuf_trans_bit_elem(in, out, size, elem_size); + return count; +} + + +/* Bitunshuffle a single block. */ +int64_t bshuf_bitunshuffle_block(ioc_chain* C_ptr, \ + const size_t size, const size_t elem_size) { + + + size_t this_iter; + const void *in; + void *out; + int64_t count; + + + + + in = ioc_get_in(C_ptr, &this_iter); + ioc_set_next_in(C_ptr, &this_iter, + (void*) ((char*) in + size * elem_size)); + out = ioc_get_out(C_ptr, &this_iter); + ioc_set_next_out(C_ptr, &this_iter, + (void *) ((char *) out + size * elem_size)); + + count = bshuf_untrans_bit_elem(in, out, size, elem_size); + return count; +} + + +/* Write a 64 bit unsigned integer to a buffer in big endian order. */ +void bshuf_write_uint64_BE(void* buf, uint64_t num) { + int ii; + uint8_t* b = (uint8_t*) buf; + uint64_t pow28 = 1 << 8; + for (ii = 7; ii >= 0; ii--) { + b[ii] = num % pow28; + num = num / pow28; + } +} + + +/* Read a 64 bit unsigned integer from a buffer big endian order. */ +uint64_t bshuf_read_uint64_BE(void* buf) { + int ii; + uint8_t* b = (uint8_t*) buf; + uint64_t num = 0, pow28 = 1 << 8, cp = 1; + for (ii = 7; ii >= 0; ii--) { + num += b[ii] * cp; + cp *= pow28; + } + return num; +} + + +/* Write a 32 bit unsigned integer to a buffer in big endian order. */ +void bshuf_write_uint32_BE(void* buf, uint32_t num) { + int ii; + uint8_t* b = (uint8_t*) buf; + uint32_t pow28 = 1 << 8; + for (ii = 3; ii >= 0; ii--) { + b[ii] = num % pow28; + num = num / pow28; + } +} + + +/* Read a 32 bit unsigned integer from a buffer big endian order. */ +uint32_t bshuf_read_uint32_BE(const void* buf) { + int ii; + uint8_t* b = (uint8_t*) buf; + uint32_t num = 0, pow28 = 1 << 8, cp = 1; + for (ii = 3; ii >= 0; ii--) { + num += b[ii] * cp; + cp *= pow28; + } + return num; +} + + +/* ---- Public functions ---- + * + * See header file for description and usage. + * + */ + +size_t bshuf_default_block_size(const size_t elem_size) { + // This function needs to be absolutely stable between versions. + // Otherwise encoded data will not be decodable. + + size_t block_size = BSHUF_TARGET_BLOCK_SIZE_B / elem_size; + // Ensure it is a required multiple. + block_size = (block_size / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT; + return MAX(block_size, BSHUF_MIN_RECOMMEND_BLOCK); +} + + +int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size, + const size_t elem_size, size_t block_size) { + + return bshuf_blocked_wrap_fun(&bshuf_bitshuffle_block, in, out, size, + elem_size, block_size); +} + + +int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size, + const size_t elem_size, size_t block_size) { + + return bshuf_blocked_wrap_fun(&bshuf_bitunshuffle_block, in, out, size, + elem_size, block_size); +} + + +#undef TRANS_BIT_8X8 +#undef TRANS_ELEM_TYPE +#undef MAX +#undef CHECK_MULT_EIGHT +#undef CHECK_ERR_FREE + +#undef USESSE2 +#undef USEAVX2 diff --git a/target/bitshuffle-0.3.4/src/bitshuffle_core.h b/target/bitshuffle-0.3.4/src/bitshuffle_core.h new file mode 100644 index 00000000..4516ef40 --- /dev/null +++ b/target/bitshuffle-0.3.4/src/bitshuffle_core.h @@ -0,0 +1,156 @@ +/* + * Bitshuffle - Filter for improving compression of typed binary data. + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + * + * Header File + * + * Worker routines return an int64_t which is the number of bytes processed + * if positive or an error code if negative. + * + * Error codes: + * -1 : Failed to allocate memory. + * -11 : Missing SSE. + * -12 : Missing AVX. + * -80 : Input size not a multiple of 8. + * -81 : block_size not multiple of 8. + * -91 : Decompression error, wrong number of bytes processed. + * -1YYY : Error internal to compression routine with error code -YYY. + */ + + +#ifndef BITSHUFFLE_CORE_H +#define BITSHUFFLE_CORE_H + +// We assume GNU g++ defining `__cplusplus` has stdint.h +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus) +#include +#else + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + typedef signed int int32_t; + typedef unsigned long long uint64_t; + typedef long long int64_t; +#endif + +#include + + +// These are usually set in the setup.py. +#ifndef BSHUF_VERSION_MAJOR +#define BSHUF_VERSION_MAJOR 0 +#define BSHUF_VERSION_MINOR 3 +#define BSHUF_VERSION_POINT 4 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* --- bshuf_using_SSE2 ---- + * + * Whether routines where compiled with the SSE2 instruction set. + * + * Returns + * ------- + * 1 if using SSE2, 0 otherwise. + * + */ +int bshuf_using_SSE2(void); + + +/* ---- bshuf_using_AVX2 ---- + * + * Whether routines where compiled with the AVX2 instruction set. + * + * Returns + * ------- + * 1 if using AVX2, 0 otherwise. + * + */ +int bshuf_using_AVX2(void); + + +/* ---- bshuf_default_block_size ---- + * + * The default block size as function of element size. + * + * This is the block size used by the blocked routines (any routine + * taking a *block_size* argument) when the block_size is not provided + * (zero is passed). + * + * The results of this routine are guaranteed to be stable such that + * shuffled/compressed data can always be decompressed. + * + * Parameters + * ---------- + * elem_size : element size of data to be shuffled/compressed. + * + */ +size_t bshuf_default_block_size(const size_t elem_size); + + +/* ---- bshuf_bitshuffle ---- + * + * Bitshuffle the data. + * + * Transpose the bits within elements, in blocks of *block_size* + * elements. + * + * Parameters + * ---------- + * in : input buffer, must be of size * elem_size bytes + * out : output buffer, must be of size * elem_size bytes + * size : number of elements in input + * elem_size : element size of typed data + * block_size : Do transpose in blocks of this many elements. Pass 0 to + * select automatically (recommended). + * + * Returns + * ------- + * number of bytes processed, negative error-code if failed. + * + */ +int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size, + const size_t elem_size, size_t block_size); + + +/* ---- bshuf_bitunshuffle ---- + * + * Unshuffle bitshuffled data. + * + * Untranspose the bits within elements, in blocks of *block_size* + * elements. + * + * To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size* + * must match the parameters used to shuffle the data. + * + * Parameters + * ---------- + * in : input buffer, must be of size * elem_size bytes + * out : output buffer, must be of size * elem_size bytes + * size : number of elements in input + * elem_size : element size of typed data + * block_size : Do transpose in blocks of this many elements. Pass 0 to + * select automatically (recommended). + * + * Returns + * ------- + * number of bytes processed, negative error-code if failed. + * + */ +int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size, + const size_t elem_size, size_t block_size); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // BITSHUFFLE_CORE_H diff --git a/target/bitshuffle-0.3.4/src/bitshuffle_internals.h b/target/bitshuffle-0.3.4/src/bitshuffle_internals.h new file mode 100644 index 00000000..e039925c --- /dev/null +++ b/target/bitshuffle-0.3.4/src/bitshuffle_internals.h @@ -0,0 +1,75 @@ +/* + * Bitshuffle - Filter for improving compression of typed binary data. + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + */ + + +#ifndef BITSHUFFLE_INTERNALS_H +#define BITSHUFFLE_INTERNALS_H + +// We assume GNU g++ defining `__cplusplus` has stdint.h +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus) +#include +#else + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + typedef signed int int32_t; + typedef unsigned long long uint64_t; + typedef long long int64_t; +#endif + +#include +#include "iochain.h" + + +// Constants. +#ifndef BSHUF_MIN_RECOMMEND_BLOCK +#define BSHUF_MIN_RECOMMEND_BLOCK 128 +#define BSHUF_BLOCKED_MULT 8 // Block sizes must be multiple of this. +#define BSHUF_TARGET_BLOCK_SIZE_B 8192 +#endif + + +// Macros. +#define CHECK_ERR_FREE(count, buf) if (count < 0) { free(buf); return count; } + + +#ifdef __cplusplus +extern "C" { +#endif + +/* ---- Utility functions for internal use only ---- */ + +int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size, + const size_t elem_size); + +/* Read a 32 bit unsigned integer from a buffer big endian order. */ +uint32_t bshuf_read_uint32_BE(const void* buf); + +/* Write a 32 bit unsigned integer to a buffer in big endian order. */ +void bshuf_write_uint32_BE(void* buf, uint32_t num); + +int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size, + const size_t elem_size); + +/* Function definition for worker functions that process a single block. */ +typedef int64_t (*bshufBlockFunDef)(ioc_chain* C_ptr, + const size_t size, const size_t elem_size); + +/* Wrap a function for processing a single block to process an entire buffer in + * parallel. */ +int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out, + const size_t size, const size_t elem_size, size_t block_size); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // BITSHUFFLE_INTERNALS_H diff --git a/target/bitshuffle-0.3.4/src/bshuf_h5filter.c b/target/bitshuffle-0.3.4/src/bshuf_h5filter.c new file mode 100644 index 00000000..f67a4a2b --- /dev/null +++ b/target/bitshuffle-0.3.4/src/bshuf_h5filter.c @@ -0,0 +1,218 @@ +/* + * Bitshuffle HDF5 filter + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + */ + +#include "bitshuffle.h" +#include "bshuf_h5filter.h" + + +#define PUSH_ERR(func, minor, str) \ + H5Epush1(__FILE__, func, __LINE__, H5E_PLINE, minor, str) + + +// Prototypes from bitshuffle.c +void bshuf_write_uint64_BE(void* buf, uint64_t num); +uint64_t bshuf_read_uint64_BE(void* buf); +void bshuf_write_uint32_BE(void* buf, uint32_t num); +uint32_t bshuf_read_uint32_BE(const void* buf); + + +// Only called on compresion, not on reverse. +herr_t bshuf_h5_set_local(hid_t dcpl, hid_t type, hid_t space){ + + herr_t r; + size_t ii; + + unsigned int elem_size; + + unsigned int flags; + size_t nelements = 8; + size_t nelem_max = 11; + unsigned values[] = {0,0,0,0,0,0,0,0,0,0,0}; + unsigned tmp_values[] = {0,0,0,0,0,0,0,0}; + char msg[80]; + + r = H5Pget_filter_by_id2(dcpl, BSHUF_H5FILTER, &flags, &nelements, + tmp_values, 0, NULL, NULL); + if(r<0) return -1; + + // First 3 slots reserved. Move any passed options to higher addresses. + for (ii=0; ii < nelements && ii + 3 < nelem_max; ii++) { + values[ii + 3] = tmp_values[ii]; + } + + nelements = 3 + nelements; + + values[0] = BSHUF_VERSION_MAJOR; + values[1] = BSHUF_VERSION_MINOR; + + elem_size = H5Tget_size(type); + if(elem_size <= 0) { + PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK, + "Invalid element size."); + return -1; + } + + values[2] = elem_size; + + // Validate user supplied arguments. + if (nelements > 3) { + if (values[3] % 8 || values[3] < 0) { + sprintf(msg, "Error in bitshuffle. Invalid block size: %d.", + values[3]); + PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK, msg); + return -1; + } + } + if (nelements > 4) { + switch (values[4]) { + case 0: + break; + case BSHUF_H5_COMPRESS_LZ4: + break; + default: + PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK, + "Invalid bitshuffle compression."); + } + } + + r = H5Pmodify_filter(dcpl, BSHUF_H5FILTER, flags, nelements, values); + if(r<0) return -1; + + return 1; +} + + +size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) { + + size_t size, elem_size; + int err; + char msg[80]; + size_t block_size = 0; + size_t buf_size_out, nbytes_uncomp, nbytes_out; + char* in_buf = *buf; + void *out_buf; + + if (cd_nelmts < 3) { + PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK, + "Not enough parameters."); + return 0; + } + elem_size = cd_values[2]; + + // User specified block size. + if (cd_nelmts > 3) block_size = cd_values[3]; + + if (block_size == 0) block_size = bshuf_default_block_size(elem_size); + + // Compression in addition to bitshiffle. + if (cd_nelmts > 4 && cd_values[4] == BSHUF_H5_COMPRESS_LZ4) { + if (flags & H5Z_FLAG_REVERSE) { + // First eight bytes is the number of bytes in the output buffer, + // little endian. + nbytes_uncomp = bshuf_read_uint64_BE(in_buf); + // Override the block size with the one read from the header. + block_size = bshuf_read_uint32_BE((const char*) in_buf + 8) / elem_size; + // Skip over the header. + in_buf += 12; + buf_size_out = nbytes_uncomp; + } else { + nbytes_uncomp = nbytes; + buf_size_out = bshuf_compress_lz4_bound(nbytes_uncomp / elem_size, + elem_size, block_size) + 12; + } + } else { + nbytes_uncomp = nbytes; + buf_size_out = nbytes; + } + + // TODO, remove this restriction by memcopying the extra. + if (nbytes_uncomp % elem_size) { + PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK, + "Non integer number of elements."); + return 0; + } + size = nbytes_uncomp / elem_size; + + out_buf = malloc(buf_size_out); + if (out_buf == NULL) { + PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK, + "Could not allocate output buffer."); + return 0; + } + + if (cd_nelmts > 4 && cd_values[4] == BSHUF_H5_COMPRESS_LZ4) { + if (flags & H5Z_FLAG_REVERSE) { + // Bit unshuffle/decompress. + err = bshuf_decompress_lz4(in_buf, out_buf, size, elem_size, block_size); + nbytes_out = nbytes_uncomp; + } else { + // Bit shuffle/compress. + // Write the header, described in + // http://www.hdfgroup.org/services/filters/HDF5_LZ4.pdf. + // Techincally we should be using signed integers instead of + // unsigned ones, however for valid inputs (positive numbers) these + // have the same representation. + bshuf_write_uint64_BE(out_buf, nbytes_uncomp); + bshuf_write_uint32_BE((char*) out_buf + 8, block_size * elem_size); + err = bshuf_compress_lz4(in_buf, (char*) out_buf + 12, size, + elem_size, block_size); nbytes_out = err + 12; } } else { + if (flags & H5Z_FLAG_REVERSE) { + // Bit unshuffle. + err = bshuf_bitunshuffle(in_buf, out_buf, size, elem_size, + block_size); } else { + // Bit shuffle. + err = bshuf_bitshuffle(in_buf, out_buf, size, elem_size, + block_size); } nbytes_out = nbytes; } + //printf("nb_in %d, nb_uncomp %d, nb_out %d, buf_out %d, block %d\n", + //nbytes, nbytes_uncomp, nbytes_out, buf_size_out, block_size); + + if (err < 0) { + sprintf(msg, "Error in bitshuffle with error code %d.", err); + PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK, msg); + free(out_buf); + return 0; + } else { + free(*buf); + *buf = out_buf; + *buf_size = buf_size_out; + + return nbytes_out; + } +} + + + +H5Z_class_t bshuf_H5Filter[1] = {{ + H5Z_CLASS_T_VERS, + (H5Z_filter_t)(BSHUF_H5FILTER), + 1, 1, + "bitshuffle; see https://github.com/kiyo-masui/bitshuffle", + NULL, + (H5Z_set_local_func_t)(bshuf_h5_set_local), + (H5Z_func_t)(bshuf_h5_filter) +}}; + + +int bshuf_register_h5filter(void){ + + int retval; + + retval = H5Zregister(bshuf_H5Filter); + if(retval<0){ + PUSH_ERR("bshuf_register_h5filter", + H5E_CANTREGISTER, "Can't register bitshuffle filter"); + } + return retval; +} + diff --git a/target/bitshuffle-0.3.4/src/bshuf_h5filter.h b/target/bitshuffle-0.3.4/src/bshuf_h5filter.h new file mode 100644 index 00000000..0a8fa6a3 --- /dev/null +++ b/target/bitshuffle-0.3.4/src/bshuf_h5filter.h @@ -0,0 +1,59 @@ +/* + * Bitshuffle HDF5 filter + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + * + * Header File + * + * Filter Options + * -------------- + * block_size (option slot 0) : interger (optional) + * What block size to use (in elements not bytes). Default is 0, + * for which bitshuffle will pick a block size with a target of 8kb. + * Compression (option slot 1) : 0 or BSHUF_H5_COMPRESS_LZ4 + * Whether to apply LZ4 compression to the data after bitshuffling. + * This is much faster than applying compression as a second filter + * because it is done when the small block of data is already in the + * L1 cache. + * + * For LZ4 compression, the compressed format of the data is the same as + * for the normal LZ4 filter described in + * http://www.hdfgroup.org/services/filters/HDF5_LZ4.pdf. + * + */ + + +#ifndef BSHUF_H5FILTER_H +#define BSHUF_H5FILTER_H + +#define H5Z_class_t_vers 2 +#include "hdf5.h" + + +#define BSHUF_H5FILTER 32008 + + +#define BSHUF_H5_COMPRESS_LZ4 2 + + +extern H5Z_class_t bshuf_H5Filter[1]; + + +/* ---- bshuf_register_h5filter ---- + * + * Register the bitshuffle HDF5 filter within the HDF5 library. + * + * Call this before using the bitshuffle HDF5 filter from C unless + * using dynamically loaded filters. + * + */ +int bshuf_register_h5filter(void); + + +#endif // BSHUF_H5FILTER_H diff --git a/target/bitshuffle-0.3.4/src/bshuf_h5plugin.c b/target/bitshuffle-0.3.4/src/bshuf_h5plugin.c new file mode 100644 index 00000000..22e99929 --- /dev/null +++ b/target/bitshuffle-0.3.4/src/bshuf_h5plugin.c @@ -0,0 +1,19 @@ +/* + * Dynamically loaded filter plugin for HDF5 Bitshuffle filter. + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + */ + + +#include "bshuf_h5filter.h" +#include "H5PLextern.h" + +H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;} +const void* H5PLget_plugin_info(void) {return bshuf_H5Filter;} + diff --git a/target/bitshuffle-0.3.4/src/iochain.c b/target/bitshuffle-0.3.4/src/iochain.c new file mode 100644 index 00000000..baa97296 --- /dev/null +++ b/target/bitshuffle-0.3.4/src/iochain.c @@ -0,0 +1,90 @@ +/* + * IOchain - Distribute a chain of dependant IO events amoung threads. + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + */ + +#include +#include "iochain.h" + + +void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0) { +#ifdef _OPENMP + omp_init_lock(&C->next_lock); + for (size_t ii = 0; ii < IOC_SIZE; ii ++) { + omp_init_lock(&(C->in_pl[ii].lock)); + omp_init_lock(&(C->out_pl[ii].lock)); + } +#endif + C->next = 0; + C->in_pl[0].ptr = in_ptr_0; + C->out_pl[0].ptr = out_ptr_0; +} + + +void ioc_destroy(ioc_chain *C) { +#ifdef _OPENMP + omp_destroy_lock(&C->next_lock); + for (size_t ii = 0; ii < IOC_SIZE; ii ++) { + omp_destroy_lock(&(C->in_pl[ii].lock)); + omp_destroy_lock(&(C->out_pl[ii].lock)); + } +#endif +} + + +const void * ioc_get_in(ioc_chain *C, size_t *this_iter) { +#ifdef _OPENMP + omp_set_lock(&C->next_lock); + #pragma omp flush +#endif + *this_iter = C->next; + C->next ++; +#ifdef _OPENMP + omp_set_lock(&(C->in_pl[*this_iter % IOC_SIZE].lock)); + omp_set_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock)); + omp_set_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock)); + omp_unset_lock(&C->next_lock); +#endif + return C->in_pl[*this_iter % IOC_SIZE].ptr; +} + + +void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr) { + C->in_pl[(*this_iter + 1) % IOC_SIZE].ptr = in_ptr; +#ifdef _OPENMP + omp_unset_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock)); +#endif +} + + +void * ioc_get_out(ioc_chain *C, size_t *this_iter) { +#ifdef _OPENMP + omp_set_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock)); + #pragma omp flush +#endif + void *out_ptr = C->out_pl[*this_iter % IOC_SIZE].ptr; +#ifdef _OPENMP + omp_unset_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock)); +#endif + return out_ptr; +} + + +void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr) { + C->out_pl[(*this_iter + 1) % IOC_SIZE].ptr = out_ptr; +#ifdef _OPENMP + omp_unset_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock)); + // *in_pl[this_iter]* lock released at the end of the iteration to avoid being + // overtaken by previous threads and having *out_pl[this_iter]* corrupted. + // Especially worried about thread 0, iteration 0. + omp_unset_lock(&(C->in_pl[(*this_iter) % IOC_SIZE].lock)); +#endif +} + diff --git a/target/bitshuffle-0.3.4/src/iochain.h b/target/bitshuffle-0.3.4/src/iochain.h new file mode 100644 index 00000000..4e225d1b --- /dev/null +++ b/target/bitshuffle-0.3.4/src/iochain.h @@ -0,0 +1,94 @@ +/* + * IOchain - Distribute a chain of dependant IO events amoung threads. + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + * + * Header File + * + * Similar in concept to a queue. Each task includes reading an input + * and writing output, but the location of the input/output (the pointers) + * depend on the previous item in the chain. + * + * This is designed for parallelizing blocked compression/decompression IO, + * where the destination of a compressed block depends on the compressed size + * of all previous blocks. + * + * Implemented with OpenMP locks. + * + * + * Usage + * ----- + * - Call `ioc_init` in serial block. + * - Each thread should create a local variable *size_t this_iter* and + * pass its address to all function calls. Its value will be set + * inside the functions and is used to identify the thread. + * - Each thread must call each of the `ioc_get*` and `ioc_set*` methods + * exactly once per iteration, starting with `ioc_get_in` and ending + * with `ioc_set_next_out`. + * - The order (`ioc_get_in`, `ioc_set_next_in`, *work*, `ioc_get_out`, + * `ioc_set_next_out`, *work*) is most efficient. + * - Have each thread call `ioc_end_pop`. + * - `ioc_get_in` is blocked until the previous entry's + * `ioc_set_next_in` is called. + * - `ioc_get_out` is blocked until the previous entry's + * `ioc_set_next_out` is called. + * - There are no blocks on the very first iteration. + * - Call `ioc_destroy` in serial block. + * - Safe for num_threads >= IOC_SIZE (but less efficient). + * + */ + + +#ifndef IOCHAIN_H +#define IOCHAIN_H + + +#include +#ifdef _OPENMP +#include +#endif + + +#define IOC_SIZE 33 + + +typedef struct ioc_ptr_and_lock { +#ifdef _OPENMP + omp_lock_t lock; +#endif + void *ptr; +} ptr_and_lock; + +typedef struct ioc_const_ptr_and_lock { +#ifdef _OPENMP + omp_lock_t lock; +#endif + const void *ptr; +} const_ptr_and_lock; + + +typedef struct ioc_chain { +#ifdef _OPENMP + omp_lock_t next_lock; +#endif + size_t next; + const_ptr_and_lock in_pl[IOC_SIZE]; + ptr_and_lock out_pl[IOC_SIZE]; +} ioc_chain; + + +void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0); +void ioc_destroy(ioc_chain *C); +const void * ioc_get_in(ioc_chain *C, size_t *this_iter); +void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr); +void * ioc_get_out(ioc_chain *C, size_t *this_iter); +void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr); + +#endif // IOCHAIN_H + diff --git a/target/bitshuffle-0.3.4/src/lzf_h5plugin.c b/target/bitshuffle-0.3.4/src/lzf_h5plugin.c new file mode 100644 index 00000000..cbf7e3d8 --- /dev/null +++ b/target/bitshuffle-0.3.4/src/lzf_h5plugin.c @@ -0,0 +1,42 @@ +/* + * Dynamically loaded filter plugin for HDF5 LZF filter. + * + * This file is part of Bitshuffle + * Author: Kiyoshi Masui + * Website: http://www.github.com/kiyo-masui/bitshuffle + * Created: 2014 + * + * See LICENSE file for details about copyright and rights to use. + * + */ + + +#define H5Z_class_t_vers 2 +#include "lzf_filter.h" +#include "H5PLextern.h" + +#include + + +size_t lzf_filter(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + + +herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space); + + +H5Z_class_t lzf_H5Filter[1] = {{ + H5Z_CLASS_T_VERS, + (H5Z_filter_t)(H5PY_FILTER_LZF), + 1, 1, + "lzf", + NULL, + (H5Z_set_local_func_t)(lzf_set_local), + (H5Z_func_t)(lzf_filter) +}}; + + +H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;} +const void* H5PLget_plugin_info(void) {return lzf_H5Filter;} + diff --git a/target/snappy-1.1.10/AUTHORS b/target/snappy-1.1.10/AUTHORS new file mode 100644 index 00000000..4858b377 --- /dev/null +++ b/target/snappy-1.1.10/AUTHORS @@ -0,0 +1 @@ +opensource@google.com diff --git a/target/snappy-1.1.10/CMakeLists.txt b/target/snappy-1.1.10/CMakeLists.txt new file mode 100644 index 00000000..68686f78 --- /dev/null +++ b/target/snappy-1.1.10/CMakeLists.txt @@ -0,0 +1,297 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +cmake_minimum_required(VERSION 3.1) +project(Snappy VERSION 1.1.8 LANGUAGES C CXX) + +# C++ standard can be overridden when this is used as a sub-project. +if(NOT CMAKE_CXX_STANDARD) + # This project requires C++11. + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) +endif(NOT CMAKE_CXX_STANDARD) + +# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make +# it prominent in the GUI. +option(BUILD_SHARED_LIBS "Build shared libraries(DLLs)." OFF) + +option(SNAPPY_BUILD_TESTS "Build Snappy's own tests." ON) + +option(SNAPPY_FUZZING_BUILD "Build Snappy for fuzzing." OFF) + +option(SNAPPY_REQUIRE_AVX "Target processors with AVX support." OFF) + +option(SNAPPY_REQUIRE_AVX2 "Target processors with AVX2 support." OFF) + +option(SNAPPY_INSTALL "Install Snappy's header and library" ON) + +include(TestBigEndian) +test_big_endian(SNAPPY_IS_BIG_ENDIAN) + +include(CheckIncludeFile) +check_include_file("byteswap.h" HAVE_BYTESWAP_H) +check_include_file("sys/endian.h" HAVE_SYS_ENDIAN_H) +check_include_file("sys/mman.h" HAVE_SYS_MMAN_H) +check_include_file("sys/resource.h" HAVE_SYS_RESOURCE_H) +check_include_file("sys/time.h" HAVE_SYS_TIME_H) +check_include_file("sys/uio.h" HAVE_SYS_UIO_H) +check_include_file("unistd.h" HAVE_UNISTD_H) +check_include_file("windows.h" HAVE_WINDOWS_H) + +include(CheckLibraryExists) +check_library_exists(z zlibVersion "" HAVE_LIBZ) +check_library_exists(lzo2 lzo1x_1_15_compress "" HAVE_LIBLZO2) + +include(CheckCXXCompilerFlag) +CHECK_CXX_COMPILER_FLAG("/arch:AVX" HAVE_VISUAL_STUDIO_ARCH_AVX) +CHECK_CXX_COMPILER_FLAG("/arch:AVX2" HAVE_VISUAL_STUDIO_ARCH_AVX2) +CHECK_CXX_COMPILER_FLAG("-mavx" HAVE_CLANG_MAVX) +CHECK_CXX_COMPILER_FLAG("-mbmi2" HAVE_CLANG_MBMI2) +if(SNAPPY_REQUIRE_AVX2) + if(HAVE_VISUAL_STUDIO_ARCH_AVX2) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") + endif(HAVE_VISUAL_STUDIO_ARCH_AVX2) + if(HAVE_CLANG_MAVX) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") + endif(HAVE_CLANG_MAVX) + if(HAVE_CLANG_MBMI2) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mbmi2") + endif(HAVE_CLANG_MBMI2) +elseif (SNAPPY_REQUIRE_AVX) + if(HAVE_VISUAL_STUDIO_ARCH_AVX) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX") + endif(HAVE_VISUAL_STUDIO_ARCH_AVX) + if(HAVE_CLANG_MAVX) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") + endif(HAVE_CLANG_MAVX) +endif(SNAPPY_REQUIRE_AVX2) + +include(CheckCXXSourceCompiles) +check_cxx_source_compiles(" +int main() { + return __builtin_expect(0, 1); +}" HAVE_BUILTIN_EXPECT) + +check_cxx_source_compiles(" +int main() { + return __builtin_ctzll(0); +}" HAVE_BUILTIN_CTZ) + +check_cxx_source_compiles(" +#include + +int main() { + const __m128i *src = 0; + __m128i dest; + const __m128i shuffle_mask = _mm_load_si128(src); + const __m128i pattern = _mm_shuffle_epi8(_mm_loadl_epi64(src), shuffle_mask); + _mm_storeu_si128(&dest, pattern); + return 0; +}" SNAPPY_HAVE_SSSE3) + +check_cxx_source_compiles(" +#include +int main() { + return _bzhi_u32(0, 1); +}" SNAPPY_HAVE_BMI2) + +include(CheckSymbolExists) +check_symbol_exists("mmap" "sys/mman.h" HAVE_FUNC_MMAP) +check_symbol_exists("sysconf" "unistd.h" HAVE_FUNC_SYSCONF) + +find_package(GTest QUIET) +if(GTEST_FOUND) + set(HAVE_GTEST 1) +endif(GTEST_FOUND) + +find_package(Gflags QUIET) +if(GFLAGS_FOUND) + set(HAVE_GFLAGS 1) +endif(GFLAGS_FOUND) + +configure_file( + "cmake/config.h.in" + "${PROJECT_BINARY_DIR}/config.h" +) + +# We don't want to define HAVE_ macros in public headers. Instead, we use +# CMake's variable substitution with 0/1 variables, which will be seen by the +# preprocessor as constants. +set(HAVE_SYS_UIO_H_01 ${HAVE_SYS_UIO_H}) +if(NOT HAVE_SYS_UIO_H_01) + set(HAVE_SYS_UIO_H_01 0) +endif(NOT HAVE_SYS_UIO_H_01) + +if (SNAPPY_FUZZING_BUILD) + if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + message(WARNING "Fuzzing builds are only supported with Clang") + endif (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + + if(NOT CMAKE_CXX_FLAGS MATCHES "-fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + endif(NOT CMAKE_CXX_FLAGS MATCHES "-fsanitize=address") + + if(NOT CMAKE_CXX_FLAGS MATCHES "-fsanitize=fuzzer-no-link") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=fuzzer-no-link") + endif(NOT CMAKE_CXX_FLAGS MATCHES "-fsanitize=fuzzer-no-link") +endif (SNAPPY_FUZZING_BUILD) + +configure_file( + "snappy-stubs-public.h.in" + "${PROJECT_BINARY_DIR}/snappy-stubs-public.h") + +add_library(snappy "") +target_sources(snappy + PRIVATE + "snappy-internal.h" + "snappy-stubs-internal.h" + "snappy-c.cc" + "snappy-sinksource.cc" + "snappy-stubs-internal.cc" + "snappy.cc" + "${PROJECT_BINARY_DIR}/config.h" + + # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install". + $<$:PUBLIC> + $ + $ + $ + $ + $ + $ + $ + $ +) +target_include_directories(snappy + PUBLIC + $ + $ + $ +) +set_target_properties(snappy + PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR}) + +target_compile_definitions(snappy PRIVATE -DHAVE_CONFIG_H) +if(BUILD_SHARED_LIBS) + set_target_properties(snappy PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) +endif(BUILD_SHARED_LIBS) + +if(SNAPPY_BUILD_TESTS) + enable_testing() + + add_executable(snappy_unittest "") + target_sources(snappy_unittest + PRIVATE + "snappy_unittest.cc" + "snappy-test.cc" + ) + target_compile_definitions(snappy_unittest PRIVATE -DHAVE_CONFIG_H) + target_link_libraries(snappy_unittest snappy ${GFLAGS_LIBRARIES}) + + if(HAVE_LIBZ) + target_link_libraries(snappy_unittest z) + endif(HAVE_LIBZ) + if(HAVE_LIBLZO2) + target_link_libraries(snappy_unittest lzo2) + endif(HAVE_LIBLZO2) + + target_include_directories(snappy_unittest + BEFORE PRIVATE + "${PROJECT_SOURCE_DIR}" + "${GTEST_INCLUDE_DIRS}" + "${GFLAGS_INCLUDE_DIRS}" + ) + + add_test( + NAME snappy_unittest + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + COMMAND "${PROJECT_BINARY_DIR}/snappy_unittest") +endif(SNAPPY_BUILD_TESTS) + +if(SNAPPY_FUZZING_BUILD) + add_executable(snappy_compress_fuzzer "") + target_sources(snappy_compress_fuzzer + PRIVATE "snappy_compress_fuzzer.cc" + ) + target_link_libraries(snappy_compress_fuzzer snappy) + set_target_properties(snappy_compress_fuzzer + PROPERTIES LINK_FLAGS "-fsanitize=fuzzer" + ) + + add_executable(snappy_uncompress_fuzzer "") + target_sources(snappy_uncompress_fuzzer + PRIVATE "snappy_uncompress_fuzzer.cc" + ) + target_link_libraries(snappy_uncompress_fuzzer snappy) + set_target_properties(snappy_uncompress_fuzzer + PROPERTIES LINK_FLAGS "-fsanitize=fuzzer" + ) +endif(SNAPPY_FUZZING_BUILD) + +# Must be included before CMAKE_INSTALL_INCLUDEDIR is used. +include(GNUInstallDirs) + +if(SNAPPY_INSTALL) + install(TARGETS snappy + EXPORT SnappyTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + install( + FILES + "snappy-c.h" + "snappy-sinksource.h" + "snappy.h" + "${PROJECT_BINARY_DIR}/snappy-stubs-public.h" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + ) + + include(CMakePackageConfigHelpers) + configure_package_config_file( + "cmake/${PROJECT_NAME}Config.cmake.in" + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) + write_basic_package_version_file( + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake" + COMPATIBILITY SameMajorVersion + ) + install( + EXPORT SnappyTargets + NAMESPACE Snappy:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) + install( + FILES + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake" + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) +endif(SNAPPY_INSTALL) diff --git a/target/snappy-1.1.10/CONTRIBUTING.md b/target/snappy-1.1.10/CONTRIBUTING.md new file mode 100644 index 00000000..c7b84516 --- /dev/null +++ b/target/snappy-1.1.10/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution, +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +Please make sure that all the automated checks (CLA, AppVeyor, Travis) pass for +your pull requests. Pull requests whose checks fail may be ignored. diff --git a/target/snappy-1.1.10/COPYING b/target/snappy-1.1.10/COPYING new file mode 100644 index 00000000..bd0e5971 --- /dev/null +++ b/target/snappy-1.1.10/COPYING @@ -0,0 +1,54 @@ +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). diff --git a/target/snappy-1.1.10/NEWS b/target/snappy-1.1.10/NEWS new file mode 100644 index 00000000..98048dbd --- /dev/null +++ b/target/snappy-1.1.10/NEWS @@ -0,0 +1,188 @@ +Snappy v1.1.8, January 15th 2020: + + * Small performance improvements. + + * Removed snappy::string alias for std::string. + + * Improved CMake configuration. + +Snappy v1.1.7, August 24th 2017: + + * Improved CMake build support for 64-bit Linux distributions. + + * MSVC builds now use MSVC-specific intrinsics that map to clzll. + + * ARM64 (AArch64) builds use the code paths optimized for 64-bit processors. + +Snappy v1.1.6, July 12th 2017: + +This is a re-release of v1.1.5 with proper SONAME / SOVERSION values. + +Snappy v1.1.5, June 28th 2017: + +This release has broken SONAME / SOVERSION values. Users of snappy as a shared +library should avoid 1.1.5 and use 1.1.6 instead. SONAME / SOVERSION errors will +manifest as the dynamic library loader complaining that it cannot find snappy's +shared library file (libsnappy.so / libsnappy.dylib), or that the library it +found does not have the required version. 1.1.6 has the same code as 1.1.5, but +carries build configuration fixes for the issues above. + + * Add CMake build support. The autoconf build support is now deprecated, and + will be removed in the next release. + + * Add AppVeyor configuration, for Windows CI coverage. + + * Small performance improvement on little-endian PowerPC. + + * Small performance improvement on LLVM with position-independent executables. + + * Fix a few issues with various build environments. + +Snappy v1.1.4, January 25th 2017: + + * Fix a 1% performance regression when snappy is used in PIE executables. + + * Improve compression performance by 5%. + + * Improve decompression performance by 20%. + +Snappy v1.1.3, July 6th 2015: + +This is the first release to be done from GitHub, which means that +some minor things like the ChangeLog format has changed (git log +format instead of svn log). + + * Add support for Uncompress() from a Source to a Sink. + + * Various minor changes to improve MSVC support; in particular, + the unit tests now compile and run under MSVC. + + +Snappy v1.1.2, February 28th 2014: + +This is a maintenance release with no changes to the actual library +source code. + + * Stop distributing benchmark data files that have unclear + or unsuitable licensing. + + * Add support for padding chunks in the framing format. + + +Snappy v1.1.1, October 15th 2013: + + * Add support for uncompressing to iovecs (scatter I/O). + The bulk of this patch was contributed by Mohit Aron. + + * Speed up decompression by ~2%; much more so (~13-20%) on + a few benchmarks on given compilers and CPUs. + + * Fix a few issues with MSVC compilation. + + * Support truncated test data in the benchmark. + + +Snappy v1.1.0, January 18th 2013: + + * Snappy now uses 64 kB block size instead of 32 kB. On average, + this means it compresses about 3% denser (more so for some + inputs), at the same or better speeds. + + * libsnappy no longer depends on iostream. + + * Some small performance improvements in compression on x86 + (0.5–1%). + + * Various portability fixes for ARM-based platforms, for MSVC, + and for GNU/Hurd. + + +Snappy v1.0.5, February 24th 2012: + + * More speed improvements. Exactly how big will depend on + the architecture: + + - 3–10% faster decompression for the base case (x86-64). + + - ARMv7 and higher can now use unaligned accesses, + and will see about 30% faster decompression and + 20–40% faster compression. + + - 32-bit platforms (ARM and 32-bit x86) will see 2–5% + faster compression. + + These are all cumulative (e.g., ARM gets all three speedups). + + * Fixed an issue where the unit test would crash on system + with less than 256 MB address space available, + e.g. some embedded platforms. + + * Added a framing format description, for use over e.g. HTTP, + or for a command-line compressor. We do not have any + implementations of this at the current point, but there seems + to be enough of a general interest in the topic. + Also make the format description slightly clearer. + + * Remove some compile-time warnings in -Wall + (mostly signed/unsigned comparisons), for easier embedding + into projects that use -Wall -Werror. + + +Snappy v1.0.4, September 15th 2011: + + * Speeded up the decompressor somewhat; typically about 2–8% + for Core i7, in 64-bit mode (comparable for Opteron). + Somewhat more for some tests, almost no gain for others. + + * Make Snappy compile on certain platforms it didn't before + (Solaris with SunPro C++, HP-UX, AIX). + + * Correct some minor errors in the format description. + + +Snappy v1.0.3, June 2nd 2011: + + * Speeded up the decompressor somewhat; about 3-6% for Core 2, + 6-13% for Core i7, and 5-12% for Opteron (all in 64-bit mode). + + * Added compressed format documentation. This text is new, + but an earlier version from Zeev Tarantov was used as reference. + + * Only link snappy_unittest against -lz and other autodetected + libraries, not libsnappy.so (which doesn't need any such dependency). + + * Fixed some display issues in the microbenchmarks, one of which would + frequently make the test crash on GNU/Hurd. + + +Snappy v1.0.2, April 29th 2011: + + * Relicense to a BSD-type license. + + * Added C bindings, contributed by Martin Gieseking. + + * More Win32 fixes, in particular for MSVC. + + * Replace geo.protodata with a newer version. + + * Fix timing inaccuracies in the unit test when comparing Snappy + to other algorithms. + + +Snappy v1.0.1, March 25th 2011: + +This is a maintenance release, mostly containing minor fixes. +There is no new functionality. The most important fixes include: + + * The COPYING file and all licensing headers now correctly state that + Snappy is licensed under the Apache 2.0 license. + + * snappy_unittest should now compile natively under Windows, + as well as on embedded systems with no mmap(). + + * Various autotools nits have been fixed. + + +Snappy v1.0, March 17th 2011: + + * Initial version. diff --git a/target/snappy-1.1.10/README.md b/target/snappy-1.1.10/README.md new file mode 100644 index 00000000..cef40174 --- /dev/null +++ b/target/snappy-1.1.10/README.md @@ -0,0 +1,148 @@ +Snappy, a fast compressor/decompressor. + + +Introduction +============ + +Snappy is a compression/decompression library. It does not aim for maximum +compression, or compatibility with any other compression library; instead, +it aims for very high speeds and reasonable compression. For instance, +compared to the fastest mode of zlib, Snappy is an order of magnitude faster +for most inputs, but the resulting compressed files are anywhere from 20% to +100% bigger. (For more information, see "Performance", below.) + +Snappy has the following properties: + + * Fast: Compression speeds at 250 MB/sec and beyond, with no assembler code. + See "Performance" below. + * Stable: Over the last few years, Snappy has compressed and decompressed + petabytes of data in Google's production environment. The Snappy bitstream + format is stable and will not change between versions. + * Robust: The Snappy decompressor is designed not to crash in the face of + corrupted or malicious input. + * Free and open source software: Snappy is licensed under a BSD-type license. + For more information, see the included COPYING file. + +Snappy has previously been called "Zippy" in some Google presentations +and the like. + + +Performance +=========== + +Snappy is intended to be fast. On a single core of a Core i7 processor +in 64-bit mode, it compresses at about 250 MB/sec or more and decompresses at +about 500 MB/sec or more. (These numbers are for the slowest inputs in our +benchmark suite; others are much faster.) In our tests, Snappy usually +is faster than algorithms in the same class (e.g. LZO, LZF, QuickLZ, +etc.) while achieving comparable compression ratios. + +Typical compression ratios (based on the benchmark suite) are about 1.5-1.7x +for plain text, about 2-4x for HTML, and of course 1.0x for JPEGs, PNGs and +other already-compressed data. Similar numbers for zlib in its fastest mode +are 2.6-2.8x, 3-7x and 1.0x, respectively. More sophisticated algorithms are +capable of achieving yet higher compression rates, although usually at the +expense of speed. Of course, compression ratio will vary significantly with +the input. + +Although Snappy should be fairly portable, it is primarily optimized +for 64-bit x86-compatible processors, and may run slower in other environments. +In particular: + + - Snappy uses 64-bit operations in several places to process more data at + once than would otherwise be possible. + - Snappy assumes unaligned 32 and 64-bit loads and stores are cheap. + On some platforms, these must be emulated with single-byte loads + and stores, which is much slower. + - Snappy assumes little-endian throughout, and needs to byte-swap data in + several places if running on a big-endian platform. + +Experience has shown that even heavily tuned code can be improved. +Performance optimizations, whether for 64-bit x86 or other platforms, +are of course most welcome; see "Contact", below. + + +Building +======== + +You need the CMake version specified in [CMakeLists.txt](./CMakeLists.txt) +or later to build: + +```bash +mkdir build +cd build && cmake ../ && make +``` + +Usage +===== + +Note that Snappy, both the implementation and the main interface, +is written in C++. However, several third-party bindings to other languages +are available; see the [home page](docs/README.md) for more information. +Also, if you want to use Snappy from C code, you can use the included C +bindings in snappy-c.h. + +To use Snappy from your own C++ program, include the file "snappy.h" from +your calling file, and link against the compiled library. + +There are many ways to call Snappy, but the simplest possible is + +```c++ +snappy::Compress(input.data(), input.size(), &output); +``` + +and similarly + +```c++ +snappy::Uncompress(input.data(), input.size(), &output); +``` + +where "input" and "output" are both instances of std::string. + +There are other interfaces that are more flexible in various ways, including +support for custom (non-array) input sources. See the header file for more +information. + + +Tests and benchmarks +==================== + +When you compile Snappy, snappy_unittest is compiled in addition to the +library itself. You do not need it to use the compressor from your own library, +but it contains several useful components for Snappy development. + +First of all, it contains unit tests, verifying correctness on your machine in +various scenarios. If you want to change or optimize Snappy, please run the +tests to verify you have not broken anything. Note that if you have the +Google Test library installed, unit test behavior (especially failures) will be +significantly more user-friendly. You can find Google Test at + + https://github.com/google/googletest + +You probably also want the gflags library for handling of command-line flags; +you can find it at + + https://gflags.github.io/gflags/ + +In addition to the unit tests, snappy contains microbenchmarks used to +tune compression and decompression performance. These are automatically run +before the unit tests, but you can disable them using the flag +--run_microbenchmarks=false if you have gflags installed (otherwise you will +need to edit the source). + +Finally, snappy can benchmark Snappy against a few other compression libraries +(zlib, LZO, LZF, and QuickLZ), if they were detected at configure time. +To benchmark using a given file, give the compression algorithm you want to test +Snappy against (e.g. --zlib) and then a list of one or more file names on the +command line. The testdata/ directory contains the files used by the +microbenchmark, which should provide a reasonably balanced starting point for +benchmarking. (Note that baddata[1-3].snappy are not intended as benchmarks; they +are used to verify correctness in the presence of corrupted data in the unit +test.) + + +Contact +======= + +Snappy is distributed through GitHub. For the latest version, a bug tracker, +and other information, see https://github.com/google/snappy. diff --git a/target/snappy-1.1.10/cmake/SnappyConfig.cmake.in b/target/snappy-1.1.10/cmake/SnappyConfig.cmake.in new file mode 100644 index 00000000..9e7d1346 --- /dev/null +++ b/target/snappy-1.1.10/cmake/SnappyConfig.cmake.in @@ -0,0 +1,33 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/SnappyTargets.cmake") + +check_required_components(Snappy) \ No newline at end of file diff --git a/target/snappy-1.1.10/cmake/config.h.in b/target/snappy-1.1.10/cmake/config.h.in new file mode 100644 index 00000000..24f27efe --- /dev/null +++ b/target/snappy-1.1.10/cmake/config.h.in @@ -0,0 +1,62 @@ +#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_ +#define THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_ + +/* Define to 1 if the compiler supports __builtin_ctz and friends. */ +#cmakedefine HAVE_BUILTIN_CTZ 1 + +/* Define to 1 if the compiler supports __builtin_expect. */ +#cmakedefine HAVE_BUILTIN_EXPECT 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_BYTESWAP_H 1 + +/* Define to 1 if you have a definition for mmap() in . */ +#cmakedefine HAVE_FUNC_MMAP 1 + +/* Define to 1 if you have a definition for sysconf() in . */ +#cmakedefine HAVE_FUNC_SYSCONF 1 + +/* Define to 1 to use the gflags package for command-line parsing. */ +#cmakedefine HAVE_GFLAGS 1 + +/* Define to 1 if you have Google Test. */ +#cmakedefine HAVE_GTEST 1 + +/* Define to 1 if you have the `lzo2' library (-llzo2). */ +#cmakedefine HAVE_LIBLZO2 1 + +/* Define to 1 if you have the `z' library (-lz). */ +#cmakedefine HAVE_LIBZ 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_ENDIAN_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_MMAN_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_WINDOWS_H 1 + +/* Define to 1 if you target processors with SSSE3+ and have . */ +#cmakedefine01 SNAPPY_HAVE_SSSE3 + +/* Define to 1 if you target processors with BMI2+ and have . */ +#cmakedefine01 SNAPPY_HAVE_BMI2 + +/* Define to 1 if your processor stores words with the most significant byte + first (like Motorola and SPARC, unlike Intel and VAX). */ +#cmakedefine SNAPPY_IS_BIG_ENDIAN 1 + +#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_ diff --git a/target/snappy-1.1.10/docs/README.md b/target/snappy-1.1.10/docs/README.md new file mode 100644 index 00000000..d5e0e638 --- /dev/null +++ b/target/snappy-1.1.10/docs/README.md @@ -0,0 +1,72 @@ +Snappy is a compression/decompression library. It does not aim for maximum +compression, or compatibility with any other compression library; instead, it +aims for very high speeds and reasonable compression. For instance, compared +to the fastest mode of zlib, Snappy is an order of magnitude faster for most +inputs, but the resulting compressed files are anywhere from 20% to 100% +bigger. On a single core of a Core i7 processor in 64-bit mode, Snappy +compresses at about 250 MB/sec or more and decompresses at about 500 MB/sec +or more. + +Snappy is widely used inside Google, in everything from BigTable and MapReduce +to our internal RPC systems. (Snappy has previously been referred to as "Zippy" +in some presentations and the likes.) + +For more information, please see the [README](../README.md). Benchmarks against +a few other compression libraries (zlib, LZO, LZF, FastLZ, and QuickLZ) are +included in the source code distribution. The source code also contains a +[formal format specification](../format_description.txt), as well +as a specification for a [framing format](../framing_format.txt) useful for +higher-level framing and encapsulation of Snappy data, e.g. for transporting +Snappy-compressed data across HTTP in a streaming fashion. Note that the Snappy +distribution currently has no code implementing the latter, but some of the +ports do (see below). + +Snappy is written in C++, but C bindings are included, and several bindings to +other languages are maintained by third parties: + +* C#: [Snappy for .NET](http://snappy4net.codeplex.com/) (P/Invoke wrapper), + [Snappy.NET](http://snappy.angeloflogic.com/) (P/Invoke wrapper), + [Snappy.Sharp](https://github.com/jeffesp/Snappy.Sharp) (native + reimplementation) +* [C port](http://github.com/andikleen/snappy-c) +* [C++ MSVC packaging](http://snappy.angeloflogic.com/) (plus Windows binaries, + NuGet packages and command-line tool) +* Common Lisp: [Library bindings](http://flambard.github.com/thnappy/), + [native reimplementation](https://github.com/brown/snappy) +* Erlang: [esnappy](https://github.com/thekvs/esnappy), + [snappy-erlang-nif](https://github.com/fdmanana/snappy-erlang-nif) +* [Go](https://github.com/golang/snappy/) +* [Haskell](http://hackage.haskell.org/package/snappy) +* [Haxe](https://github.com/MaddinXx/hxsnappy) (C++/Neko) +* [iOS packaging](https://github.com/ideawu/snappy-ios) +* Java: [JNI wrapper](https://github.com/xerial/snappy-java) (including the + framing format), [native reimplementation](http://code.google.com/p/jsnappy/), + [other native reimplementation](https://github.com/dain/snappy) (including + the framing format) +* [Lua](https://github.com/forhappy/lua-snappy) +* [Node.js](https://github.com/kesla/node-snappy) (including the [framing + format](https://github.com/kesla/node-snappy-stream)) +* [Perl](http://search.cpan.org/dist/Compress-Snappy/) +* [PHP](https://github.com/kjdev/php-ext-snappy) +* [Python](http://pypi.python.org/pypi/python-snappy) (including a command-line + tool for the framing format) +* [R](https://github.com/lulyon/R-snappy) +* [Ruby](https://github.com/miyucy/snappy) +* [Rust](https://github.com/BurntSushi/rust-snappy) +* [Smalltalk](https://github.com/mumez/sqnappy) (including the framing format) + +Snappy is used or is available as an alternative in software such as + +* [MongoDB](https://www.mongodb.com/) +* [Cassandra](http://cassandra.apache.org/) +* [Couchbase](http://www.couchbase.com/) +* [Hadoop](http://hadoop.apache.org/) +* [LessFS](http://www.lessfs.com/wordpress/) +* [LevelDB](https://github.com/google/leveldb) (which is in turn used by + [Google Chrome](http://chrome.google.com/)) +* [Lucene](http://lucene.apache.org/) +* [VoltDB](http://voltdb.com/) + +If you know of more, do not hesitate to let us know. The easiest way to get in +touch is via the +[Snappy discussion mailing list](http://groups.google.com/group/snappy-compression). diff --git a/target/snappy-1.1.10/format_description.txt b/target/snappy-1.1.10/format_description.txt new file mode 100644 index 00000000..20db66c1 --- /dev/null +++ b/target/snappy-1.1.10/format_description.txt @@ -0,0 +1,110 @@ +Snappy compressed format description +Last revised: 2011-10-05 + + +This is not a formal specification, but should suffice to explain most +relevant parts of how the Snappy format works. It is originally based on +text by Zeev Tarantov. + +Snappy is a LZ77-type compressor with a fixed, byte-oriented encoding. +There is no entropy encoder backend nor framing layer -- the latter is +assumed to be handled by other parts of the system. + +This document only describes the format, not how the Snappy compressor nor +decompressor actually works. The correctness of the decompressor should not +depend on implementation details of the compressor, and vice versa. + + +1. Preamble + +The stream starts with the uncompressed length (up to a maximum of 2^32 - 1), +stored as a little-endian varint. Varints consist of a series of bytes, +where the lower 7 bits are data and the upper bit is set iff there are +more bytes to be read. In other words, an uncompressed length of 64 would +be stored as 0x40, and an uncompressed length of 2097150 (0x1FFFFE) +would be stored as 0xFE 0xFF 0x7F. + + +2. The compressed stream itself + +There are two types of elements in a Snappy stream: Literals and +copies (backreferences). There is no restriction on the order of elements, +except that the stream naturally cannot start with a copy. (Having +two literals in a row is never optimal from a compression point of +view, but nevertheless fully permitted.) Each element starts with a tag byte, +and the lower two bits of this tag byte signal what type of element will +follow: + + 00: Literal + 01: Copy with 1-byte offset + 10: Copy with 2-byte offset + 11: Copy with 4-byte offset + +The interpretation of the upper six bits are element-dependent. + + +2.1. Literals (00) + +Literals are uncompressed data stored directly in the byte stream. +The literal length is stored differently depending on the length +of the literal: + + - For literals up to and including 60 bytes in length, the upper + six bits of the tag byte contain (len-1). The literal follows + immediately thereafter in the bytestream. + - For longer literals, the (len-1) value is stored after the tag byte, + little-endian. The upper six bits of the tag byte describe how + many bytes are used for the length; 60, 61, 62 or 63 for + 1-4 bytes, respectively. The literal itself follows after the + length. + + +2.2. Copies + +Copies are references back into previous decompressed data, telling +the decompressor to reuse data it has previously decoded. +They encode two values: The _offset_, saying how many bytes back +from the current position to read, and the _length_, how many bytes +to copy. Offsets of zero can be encoded, but are not legal; +similarly, it is possible to encode backreferences that would +go past the end of the block (offset > current decompressed position), +which is also nonsensical and thus not allowed. + +As in most LZ77-based compressors, the length can be larger than the offset, +yielding a form of run-length encoding (RLE). For instance, +"xababab" could be encoded as + + + +Note that since the current Snappy compressor works in 32 kB +blocks and does not do matching across blocks, it will never produce +a bitstream with offsets larger than about 32768. However, the +decompressor should not rely on this, as it may change in the future. + +There are several different kinds of copy elements, depending on +the amount of bytes to be copied (length), and how far back the +data to be copied is (offset). + + +2.2.1. Copy with 1-byte offset (01) + +These elements can encode lengths between [4..11] bytes and offsets +between [0..2047] bytes. (len-4) occupies three bits and is stored +in bits [2..4] of the tag byte. The offset occupies 11 bits, of which the +upper three are stored in the upper three bits ([5..7]) of the tag byte, +and the lower eight are stored in a byte following the tag byte. + + +2.2.2. Copy with 2-byte offset (10) + +These elements can encode lengths between [1..64] and offsets from +[0..65535]. (len-1) occupies six bits and is stored in the upper +six bits ([2..7]) of the tag byte. The offset is stored as a +little-endian 16-bit integer in the two bytes following the tag byte. + + +2.2.3. Copy with 4-byte offset (11) + +These are like the copies with 2-byte offsets (see previous subsection), +except that the offset is stored as a 32-bit integer instead of a +16-bit integer (and thus will occupy four bytes). diff --git a/target/snappy-1.1.10/framing_format.txt b/target/snappy-1.1.10/framing_format.txt new file mode 100644 index 00000000..9764e83d --- /dev/null +++ b/target/snappy-1.1.10/framing_format.txt @@ -0,0 +1,135 @@ +Snappy framing format description +Last revised: 2013-10-25 + +This format decribes a framing format for Snappy, allowing compressing to +files or streams that can then more easily be decompressed without having +to hold the entire stream in memory. It also provides data checksums to +help verify integrity. It does not provide metadata checksums, so it does +not protect against e.g. all forms of truncations. + +Implementation of the framing format is optional for Snappy compressors and +decompressor; it is not part of the Snappy core specification. + + +1. General structure + +The file consists solely of chunks, lying back-to-back with no padding +in between. Each chunk consists first a single byte of chunk identifier, +then a three-byte little-endian length of the chunk in bytes (from 0 to +16777215, inclusive), and then the data if any. The four bytes of chunk +header is not counted in the data length. + +The different chunk types are listed below. The first chunk must always +be the stream identifier chunk (see section 4.1, below). The stream +ends when the file ends -- there is no explicit end-of-file marker. + + +2. File type identification + +The following identifiers for this format are recommended where appropriate. +However, note that none have been registered officially, so this is only to +be taken as a guideline. We use "Snappy framed" to distinguish between this +format and raw Snappy data. + + File extension: .sz + MIME type: application/x-snappy-framed + HTTP Content-Encoding: x-snappy-framed + + +3. Checksum format + +Some chunks have data protected by a checksum (the ones that do will say so +explicitly). The checksums are always masked CRC-32Cs. + +A description of CRC-32C can be found in RFC 3720, section 12.1, with +examples in section B.4. + +Checksums are not stored directly, but masked, as checksumming data and +then its own checksum can be problematic. The masking is the same as used +in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant +0xa282ead8 (using wraparound as normal for unsigned integers). This is +equivalent to the following C code: + + uint32_t mask_checksum(uint32_t x) { + return ((x >> 15) | (x << 17)) + 0xa282ead8; + } + +Note that the masking is reversible. + +The checksum is always stored as a four bytes long integer, in little-endian. + + +4. Chunk types + +The currently supported chunk types are described below. The list may +be extended in the future. + + +4.1. Stream identifier (chunk type 0xff) + +The stream identifier is always the first element in the stream. +It is exactly six bytes long and contains "sNaPpY" in ASCII. This means that +a valid Snappy framed stream always starts with the bytes + + 0xff 0x06 0x00 0x00 0x73 0x4e 0x61 0x50 0x70 0x59 + +The stream identifier chunk can come multiple times in the stream besides +the first; if such a chunk shows up, it should simply be ignored, assuming +it has the right length and contents. This allows for easy concatenation of +compressed files without the need for re-framing. + + +4.2. Compressed data (chunk type 0x00) + +Compressed data chunks contain a normal Snappy compressed bitstream; +see the compressed format specification. The compressed data is preceded by +the CRC-32C (see section 3) of the _uncompressed_ data. + +Note that the data portion of the chunk, i.e., the compressed contents, +can be at most 16777211 bytes (2^24 - 1, minus the checksum). +However, we place an additional restriction that the uncompressed data +in a chunk must be no longer than 65536 bytes. This allows consumers to +easily use small fixed-size buffers. + + +4.3. Uncompressed data (chunk type 0x01) + +Uncompressed data chunks allow a compressor to send uncompressed, +raw data; this is useful if, for instance, uncompressible or +near-incompressible data is detected, and faster decompression is desired. + +As in the compressed chunks, the data is preceded by its own masked +CRC-32C (see section 3). + +An uncompressed data chunk, like compressed data chunks, should contain +no more than 65536 data bytes, so the maximum legal chunk length with the +checksum is 65540. + + +4.4. Padding (chunk type 0xfe) + +Padding chunks allow a compressor to increase the size of the data stream +so that it complies with external demands, e.g. that the total number of +bytes is a multiple of some value. + +All bytes of the padding chunk, except the chunk byte itself and the length, +should be zero, but decompressors must not try to interpret or verify the +padding data in any way. + + +4.5. Reserved unskippable chunks (chunk types 0x02-0x7f) + +These are reserved for future expansion. A decoder that sees such a chunk +should immediately return an error, as it must assume it cannot decode the +stream correctly. + +Future versions of this specification may define meanings for these chunks. + + +4.6. Reserved skippable chunks (chunk types 0x80-0xfd) + +These are also reserved for future expansion, but unlike the chunks +described in 4.5, a decoder seeing these must skip them and continue +decoding. + +Future versions of this specification may define meanings for these chunks. diff --git a/target/snappy-1.1.10/snappy-c.cc b/target/snappy-1.1.10/snappy-c.cc new file mode 100644 index 00000000..473a0b09 --- /dev/null +++ b/target/snappy-1.1.10/snappy-c.cc @@ -0,0 +1,90 @@ +// Copyright 2011 Martin Gieseking . +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "snappy.h" +#include "snappy-c.h" + +extern "C" { + +snappy_status snappy_compress(const char* input, + size_t input_length, + char* compressed, + size_t *compressed_length) { + if (*compressed_length < snappy_max_compressed_length(input_length)) { + return SNAPPY_BUFFER_TOO_SMALL; + } + snappy::RawCompress(input, input_length, compressed, compressed_length); + return SNAPPY_OK; +} + +snappy_status snappy_uncompress(const char* compressed, + size_t compressed_length, + char* uncompressed, + size_t* uncompressed_length) { + size_t real_uncompressed_length; + if (!snappy::GetUncompressedLength(compressed, + compressed_length, + &real_uncompressed_length)) { + return SNAPPY_INVALID_INPUT; + } + if (*uncompressed_length < real_uncompressed_length) { + return SNAPPY_BUFFER_TOO_SMALL; + } + if (!snappy::RawUncompress(compressed, compressed_length, uncompressed)) { + return SNAPPY_INVALID_INPUT; + } + *uncompressed_length = real_uncompressed_length; + return SNAPPY_OK; +} + +size_t snappy_max_compressed_length(size_t source_length) { + return snappy::MaxCompressedLength(source_length); +} + +snappy_status snappy_uncompressed_length(const char *compressed, + size_t compressed_length, + size_t *result) { + if (snappy::GetUncompressedLength(compressed, + compressed_length, + result)) { + return SNAPPY_OK; + } else { + return SNAPPY_INVALID_INPUT; + } +} + +snappy_status snappy_validate_compressed_buffer(const char *compressed, + size_t compressed_length) { + if (snappy::IsValidCompressedBuffer(compressed, compressed_length)) { + return SNAPPY_OK; + } else { + return SNAPPY_INVALID_INPUT; + } +} + +} // extern "C" diff --git a/target/snappy-1.1.10/snappy-c.h b/target/snappy-1.1.10/snappy-c.h new file mode 100644 index 00000000..32aa0c6b --- /dev/null +++ b/target/snappy-1.1.10/snappy-c.h @@ -0,0 +1,138 @@ +/* + * Copyright 2011 Martin Gieseking . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Plain C interface (a wrapper around the C++ implementation). + */ + +#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_ +#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* + * Return values; see the documentation for each function to know + * what each can return. + */ +typedef enum { + SNAPPY_OK = 0, + SNAPPY_INVALID_INPUT = 1, + SNAPPY_BUFFER_TOO_SMALL = 2 +} snappy_status; + +/* + * Takes the data stored in "input[0..input_length-1]" and stores + * it in the array pointed to by "compressed". + * + * signals the space available in "compressed". + * If it is not at least equal to "snappy_max_compressed_length(input_length)", + * SNAPPY_BUFFER_TOO_SMALL is returned. After successful compression, + * contains the true length of the compressed output, + * and SNAPPY_OK is returned. + * + * Example: + * size_t output_length = snappy_max_compressed_length(input_length); + * char* output = (char*)malloc(output_length); + * if (snappy_compress(input, input_length, output, &output_length) + * == SNAPPY_OK) { + * ... Process(output, output_length) ... + * } + * free(output); + */ +snappy_status snappy_compress(const char* input, + size_t input_length, + char* compressed, + size_t* compressed_length); + +/* + * Given data in "compressed[0..compressed_length-1]" generated by + * calling the snappy_compress routine, this routine stores + * the uncompressed data to + * uncompressed[0..uncompressed_length-1]. + * Returns failure (a value not equal to SNAPPY_OK) if the message + * is corrupted and could not be decrypted. + * + * signals the space available in "uncompressed". + * If it is not at least equal to the value returned by + * snappy_uncompressed_length for this stream, SNAPPY_BUFFER_TOO_SMALL + * is returned. After successful decompression, + * contains the true length of the decompressed output. + * + * Example: + * size_t output_length; + * if (snappy_uncompressed_length(input, input_length, &output_length) + * != SNAPPY_OK) { + * ... fail ... + * } + * char* output = (char*)malloc(output_length); + * if (snappy_uncompress(input, input_length, output, &output_length) + * == SNAPPY_OK) { + * ... Process(output, output_length) ... + * } + * free(output); + */ +snappy_status snappy_uncompress(const char* compressed, + size_t compressed_length, + char* uncompressed, + size_t* uncompressed_length); + +/* + * Returns the maximal size of the compressed representation of + * input data that is "source_length" bytes in length. + */ +size_t snappy_max_compressed_length(size_t source_length); + +/* + * REQUIRES: "compressed[]" was produced by snappy_compress() + * Returns SNAPPY_OK and stores the length of the uncompressed data in + * *result normally. Returns SNAPPY_INVALID_INPUT on parsing error. + * This operation takes O(1) time. + */ +snappy_status snappy_uncompressed_length(const char* compressed, + size_t compressed_length, + size_t* result); + +/* + * Check if the contents of "compressed[]" can be uncompressed successfully. + * Does not return the uncompressed data; if so, returns SNAPPY_OK, + * or if not, returns SNAPPY_INVALID_INPUT. + * Takes time proportional to compressed_length, but is usually at least a + * factor of four faster than actual decompression. + */ +snappy_status snappy_validate_compressed_buffer(const char* compressed, + size_t compressed_length); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */ diff --git a/target/snappy-1.1.10/snappy-internal.h b/target/snappy-1.1.10/snappy-internal.h new file mode 100644 index 00000000..1e1c307f --- /dev/null +++ b/target/snappy-1.1.10/snappy-internal.h @@ -0,0 +1,231 @@ +// Copyright 2008 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Internals shared between the Snappy implementation and its unittest. + +#ifndef THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_ +#define THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_ + +#include "snappy-stubs-internal.h" + +namespace snappy { +namespace internal { + +// Working memory performs a single allocation to hold all scratch space +// required for compression. +class WorkingMemory { + public: + explicit WorkingMemory(size_t input_size); + ~WorkingMemory(); + + // Allocates and clears a hash table using memory in "*this", + // stores the number of buckets in "*table_size" and returns a pointer to + // the base of the hash table. + uint16* GetHashTable(size_t fragment_size, int* table_size) const; + char* GetScratchInput() const { return input_; } + char* GetScratchOutput() const { return output_; } + + private: + char* mem_; // the allocated memory, never nullptr + size_t size_; // the size of the allocated memory, never 0 + uint16* table_; // the pointer to the hashtable + char* input_; // the pointer to the input scratch buffer + char* output_; // the pointer to the output scratch buffer + + // No copying + WorkingMemory(const WorkingMemory&); + void operator=(const WorkingMemory&); +}; + +// Flat array compression that does not emit the "uncompressed length" +// prefix. Compresses "input" string to the "*op" buffer. +// +// REQUIRES: "input_length <= kBlockSize" +// REQUIRES: "op" points to an array of memory that is at least +// "MaxCompressedLength(input_length)" in size. +// REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. +// REQUIRES: "table_size" is a power of two +// +// Returns an "end" pointer into "op" buffer. +// "end - op" is the compressed size of "input". +char* CompressFragment(const char* input, + size_t input_length, + char* op, + uint16* table, + const int table_size); + +// Find the largest n such that +// +// s1[0,n-1] == s2[0,n-1] +// and n <= (s2_limit - s2). +// +// Return make_pair(n, n < 8). +// Does not read *s2_limit or beyond. +// Does not read *(s1 + (s2_limit - s2)) or beyond. +// Requires that s2_limit >= s2. +// +// Separate implementation for 64-bit, little-endian cpus. +#if !defined(SNAPPY_IS_BIG_ENDIAN) && \ + (defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)) +static inline std::pair FindMatchLength(const char* s1, + const char* s2, + const char* s2_limit) { + assert(s2_limit >= s2); + size_t matched = 0; + + // This block isn't necessary for correctness; we could just start looping + // immediately. As an optimization though, it is useful. It creates some not + // uncommon code paths that determine, without extra effort, whether the match + // length is less than 8. In short, we are hoping to avoid a conditional + // branch, and perhaps get better code layout from the C++ compiler. + if (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) { + uint64 a1 = UNALIGNED_LOAD64(s1); + uint64 a2 = UNALIGNED_LOAD64(s2); + if (a1 != a2) { + return std::pair(Bits::FindLSBSetNonZero64(a1 ^ a2) >> 3, + true); + } else { + matched = 8; + s2 += 8; + } + } + + // Find out how long the match is. We loop over the data 64 bits at a + // time until we find a 64-bit block that doesn't match; then we find + // the first non-matching bit and use that to calculate the total + // length of the match. + while (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) { + if (UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched)) { + s2 += 8; + matched += 8; + } else { + uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched); + int matching_bits = Bits::FindLSBSetNonZero64(x); + matched += matching_bits >> 3; + assert(matched >= 8); + return std::pair(matched, false); + } + } + while (SNAPPY_PREDICT_TRUE(s2 < s2_limit)) { + if (s1[matched] == *s2) { + ++s2; + ++matched; + } else { + return std::pair(matched, matched < 8); + } + } + return std::pair(matched, matched < 8); +} +#else +static inline std::pair FindMatchLength(const char* s1, + const char* s2, + const char* s2_limit) { + // Implementation based on the x86-64 version, above. + assert(s2_limit >= s2); + int matched = 0; + + while (s2 <= s2_limit - 4 && + UNALIGNED_LOAD32(s2) == UNALIGNED_LOAD32(s1 + matched)) { + s2 += 4; + matched += 4; + } + if (LittleEndian::IsLittleEndian() && s2 <= s2_limit - 4) { + uint32 x = UNALIGNED_LOAD32(s2) ^ UNALIGNED_LOAD32(s1 + matched); + int matching_bits = Bits::FindLSBSetNonZero(x); + matched += matching_bits >> 3; + } else { + while ((s2 < s2_limit) && (s1[matched] == *s2)) { + ++s2; + ++matched; + } + } + return std::pair(matched, matched < 8); +} +#endif + +// Lookup tables for decompression code. Give --snappy_dump_decompression_table +// to the unit test to recompute char_table. + +enum { + LITERAL = 0, + COPY_1_BYTE_OFFSET = 1, // 3 bit length + 3 bits of offset in opcode + COPY_2_BYTE_OFFSET = 2, + COPY_4_BYTE_OFFSET = 3 +}; +static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset. + +// Data stored per entry in lookup table: +// Range Bits-used Description +// ------------------------------------ +// 1..64 0..7 Literal/copy length encoded in opcode byte +// 0..7 8..10 Copy offset encoded in opcode byte / 256 +// 0..4 11..13 Extra bytes after opcode +// +// We use eight bits for the length even though 7 would have sufficed +// because of efficiency reasons: +// (1) Extracting a byte is faster than a bit-field +// (2) It properly aligns copy offset so we do not need a <<8 +static const uint16 char_table[256] = { + 0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002, + 0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004, + 0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006, + 0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008, + 0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a, + 0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c, + 0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e, + 0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010, + 0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012, + 0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014, + 0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016, + 0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018, + 0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a, + 0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c, + 0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e, + 0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020, + 0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022, + 0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024, + 0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026, + 0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028, + 0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a, + 0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c, + 0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e, + 0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030, + 0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032, + 0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034, + 0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036, + 0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038, + 0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a, + 0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c, + 0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e, + 0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040 +}; + +} // end namespace internal +} // end namespace snappy + +#endif // THIRD_PARTY_SNAPPY_SNAPPY_INTERNAL_H_ diff --git a/target/snappy-1.1.10/snappy-sinksource.cc b/target/snappy-1.1.10/snappy-sinksource.cc new file mode 100644 index 00000000..369a1321 --- /dev/null +++ b/target/snappy-1.1.10/snappy-sinksource.cc @@ -0,0 +1,104 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include "snappy-sinksource.h" + +namespace snappy { + +Source::~Source() { } + +Sink::~Sink() { } + +char* Sink::GetAppendBuffer(size_t length, char* scratch) { + return scratch; +} + +char* Sink::GetAppendBufferVariable( + size_t min_size, size_t desired_size_hint, char* scratch, + size_t scratch_size, size_t* allocated_size) { + *allocated_size = scratch_size; + return scratch; +} + +void Sink::AppendAndTakeOwnership( + char* bytes, size_t n, + void (*deleter)(void*, const char*, size_t), + void *deleter_arg) { + Append(bytes, n); + (*deleter)(deleter_arg, bytes, n); +} + +ByteArraySource::~ByteArraySource() { } + +size_t ByteArraySource::Available() const { return left_; } + +const char* ByteArraySource::Peek(size_t* len) { + *len = left_; + return ptr_; +} + +void ByteArraySource::Skip(size_t n) { + left_ -= n; + ptr_ += n; +} + +UncheckedByteArraySink::~UncheckedByteArraySink() { } + +void UncheckedByteArraySink::Append(const char* data, size_t n) { + // Do no copying if the caller filled in the result of GetAppendBuffer() + if (data != dest_) { + memcpy(dest_, data, n); + } + dest_ += n; +} + +char* UncheckedByteArraySink::GetAppendBuffer(size_t len, char* scratch) { + return dest_; +} + +void UncheckedByteArraySink::AppendAndTakeOwnership( + char* data, size_t n, + void (*deleter)(void*, const char*, size_t), + void *deleter_arg) { + if (data != dest_) { + memcpy(dest_, data, n); + (*deleter)(deleter_arg, data, n); + } + dest_ += n; +} + +char* UncheckedByteArraySink::GetAppendBufferVariable( + size_t min_size, size_t desired_size_hint, char* scratch, + size_t scratch_size, size_t* allocated_size) { + *allocated_size = desired_size_hint; + return dest_; +} + +} // namespace snappy diff --git a/target/snappy-1.1.10/snappy-sinksource.h b/target/snappy-1.1.10/snappy-sinksource.h new file mode 100644 index 00000000..8afcdaaa --- /dev/null +++ b/target/snappy-1.1.10/snappy-sinksource.h @@ -0,0 +1,182 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_ +#define THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_ + +#include + +namespace snappy { + +// A Sink is an interface that consumes a sequence of bytes. +class Sink { + public: + Sink() { } + virtual ~Sink(); + + // Append "bytes[0,n-1]" to this. + virtual void Append(const char* bytes, size_t n) = 0; + + // Returns a writable buffer of the specified length for appending. + // May return a pointer to the caller-owned scratch buffer which + // must have at least the indicated length. The returned buffer is + // only valid until the next operation on this Sink. + // + // After writing at most "length" bytes, call Append() with the + // pointer returned from this function and the number of bytes + // written. Many Append() implementations will avoid copying + // bytes if this function returned an internal buffer. + // + // If a non-scratch buffer is returned, the caller may only pass a + // prefix of it to Append(). That is, it is not correct to pass an + // interior pointer of the returned array to Append(). + // + // The default implementation always returns the scratch buffer. + virtual char* GetAppendBuffer(size_t length, char* scratch); + + // For higher performance, Sink implementations can provide custom + // AppendAndTakeOwnership() and GetAppendBufferVariable() methods. + // These methods can reduce the number of copies done during + // compression/decompression. + + // Append "bytes[0,n-1] to the sink. Takes ownership of "bytes" + // and calls the deleter function as (*deleter)(deleter_arg, bytes, n) + // to free the buffer. deleter function must be non NULL. + // + // The default implementation just calls Append and frees "bytes". + // Other implementations may avoid a copy while appending the buffer. + virtual void AppendAndTakeOwnership( + char* bytes, size_t n, void (*deleter)(void*, const char*, size_t), + void *deleter_arg); + + // Returns a writable buffer for appending and writes the buffer's capacity to + // *allocated_size. Guarantees *allocated_size >= min_size. + // May return a pointer to the caller-owned scratch buffer which must have + // scratch_size >= min_size. + // + // The returned buffer is only valid until the next operation + // on this ByteSink. + // + // After writing at most *allocated_size bytes, call Append() with the + // pointer returned from this function and the number of bytes written. + // Many Append() implementations will avoid copying bytes if this function + // returned an internal buffer. + // + // If the sink implementation allocates or reallocates an internal buffer, + // it should use the desired_size_hint if appropriate. If a caller cannot + // provide a reasonable guess at the desired capacity, it should set + // desired_size_hint = 0. + // + // If a non-scratch buffer is returned, the caller may only pass + // a prefix to it to Append(). That is, it is not correct to pass an + // interior pointer to Append(). + // + // The default implementation always returns the scratch buffer. + virtual char* GetAppendBufferVariable( + size_t min_size, size_t desired_size_hint, char* scratch, + size_t scratch_size, size_t* allocated_size); + + private: + // No copying + Sink(const Sink&); + void operator=(const Sink&); +}; + +// A Source is an interface that yields a sequence of bytes +class Source { + public: + Source() { } + virtual ~Source(); + + // Return the number of bytes left to read from the source + virtual size_t Available() const = 0; + + // Peek at the next flat region of the source. Does not reposition + // the source. The returned region is empty iff Available()==0. + // + // Returns a pointer to the beginning of the region and store its + // length in *len. + // + // The returned region is valid until the next call to Skip() or + // until this object is destroyed, whichever occurs first. + // + // The returned region may be larger than Available() (for example + // if this ByteSource is a view on a substring of a larger source). + // The caller is responsible for ensuring that it only reads the + // Available() bytes. + virtual const char* Peek(size_t* len) = 0; + + // Skip the next n bytes. Invalidates any buffer returned by + // a previous call to Peek(). + // REQUIRES: Available() >= n + virtual void Skip(size_t n) = 0; + + private: + // No copying + Source(const Source&); + void operator=(const Source&); +}; + +// A Source implementation that yields the contents of a flat array +class ByteArraySource : public Source { + public: + ByteArraySource(const char* p, size_t n) : ptr_(p), left_(n) { } + virtual ~ByteArraySource(); + virtual size_t Available() const; + virtual const char* Peek(size_t* len); + virtual void Skip(size_t n); + private: + const char* ptr_; + size_t left_; +}; + +// A Sink implementation that writes to a flat array without any bound checks. +class UncheckedByteArraySink : public Sink { + public: + explicit UncheckedByteArraySink(char* dest) : dest_(dest) { } + virtual ~UncheckedByteArraySink(); + virtual void Append(const char* data, size_t n); + virtual char* GetAppendBuffer(size_t len, char* scratch); + virtual char* GetAppendBufferVariable( + size_t min_size, size_t desired_size_hint, char* scratch, + size_t scratch_size, size_t* allocated_size); + virtual void AppendAndTakeOwnership( + char* bytes, size_t n, void (*deleter)(void*, const char*, size_t), + void *deleter_arg); + + // Return the current output pointer so that a caller can see how + // many bytes were produced. + // Note: this is not a Sink method. + char* CurrentDestination() const { return dest_; } + private: + char* dest_; +}; + +} // namespace snappy + +#endif // THIRD_PARTY_SNAPPY_SNAPPY_SINKSOURCE_H_ diff --git a/target/snappy-1.1.10/snappy-stubs-internal.cc b/target/snappy-1.1.10/snappy-stubs-internal.cc new file mode 100644 index 00000000..66ed2e90 --- /dev/null +++ b/target/snappy-1.1.10/snappy-stubs-internal.cc @@ -0,0 +1,42 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include + +#include "snappy-stubs-internal.h" + +namespace snappy { + +void Varint::Append32(std::string* s, uint32 value) { + char buf[Varint::kMax32]; + const char* p = Varint::Encode32(buf, value); + s->append(buf, p - buf); +} + +} // namespace snappy diff --git a/target/snappy-1.1.10/snappy-stubs-internal.h b/target/snappy-1.1.10/snappy-stubs-internal.h new file mode 100644 index 00000000..4854689d --- /dev/null +++ b/target/snappy-1.1.10/snappy-stubs-internal.h @@ -0,0 +1,606 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Various stubs for the open-source version of Snappy. + +#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ +#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#include +#include + +#ifdef HAVE_SYS_MMAN_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif + +#if defined(_MSC_VER) +#include +#endif // defined(_MSC_VER) + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if __has_feature(memory_sanitizer) +#include +#define SNAPPY_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \ + __msan_unpoison((address), (size)) +#else +#define SNAPPY_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) /* empty */ +#endif // __has_feature(memory_sanitizer) + +#include "snappy-stubs-public.h" + +#if defined(__x86_64__) + +// Enable 64-bit optimized versions of some routines. +#define ARCH_K8 1 + +#elif defined(__ppc64__) + +#define ARCH_PPC 1 + +#elif defined(__aarch64__) + +#define ARCH_ARM 1 + +#endif + +// Needed by OS X, among others. +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +// The size of an array, if known at compile-time. +// Will give unexpected results if used on a pointer. +// We undefine it first, since some compilers already have a definition. +#ifdef ARRAYSIZE +#undef ARRAYSIZE +#endif +#define ARRAYSIZE(a) (sizeof(a) / sizeof(*(a))) + +// Static prediction hints. +#ifdef HAVE_BUILTIN_EXPECT +#define SNAPPY_PREDICT_FALSE(x) (__builtin_expect(x, 0)) +#define SNAPPY_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) +#else +#define SNAPPY_PREDICT_FALSE(x) x +#define SNAPPY_PREDICT_TRUE(x) x +#endif + +// This is only used for recomputing the tag byte table used during +// decompression; for simplicity we just remove it from the open-source +// version (anyone who wants to regenerate it can just do the call +// themselves within main()). +#define DEFINE_bool(flag_name, default_value, description) \ + bool FLAGS_ ## flag_name = default_value +#define DECLARE_bool(flag_name) \ + extern bool FLAGS_ ## flag_name + +namespace snappy { + +static const uint32 kuint32max = static_cast(0xFFFFFFFF); +static const int64 kint64max = static_cast(0x7FFFFFFFFFFFFFFFLL); + +// Potentially unaligned loads and stores. + +// x86, PowerPC, and ARM64 can simply do these loads and stores native. + +#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || \ + defined(__aarch64__) + +#define UNALIGNED_LOAD16(_p) (*reinterpret_cast(_p)) +#define UNALIGNED_LOAD32(_p) (*reinterpret_cast(_p)) +#define UNALIGNED_LOAD64(_p) (*reinterpret_cast(_p)) + +#define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast(_p) = (_val)) +#define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast(_p) = (_val)) +#define UNALIGNED_STORE64(_p, _val) (*reinterpret_cast(_p) = (_val)) + +// ARMv7 and newer support native unaligned accesses, but only of 16-bit +// and 32-bit values (not 64-bit); older versions either raise a fatal signal, +// do an unaligned read and rotate the words around a bit, or do the reads very +// slowly (trip through kernel mode). There's no simple #define that says just +// “ARMv7 or higher”, so we have to filter away all ARMv5 and ARMv6 +// sub-architectures. +// +// This is a mess, but there's not much we can do about it. +// +// To further complicate matters, only LDR instructions (single reads) are +// allowed to be unaligned, not LDRD (two reads) or LDM (many reads). Unless we +// explicitly tell the compiler that these accesses can be unaligned, it can and +// will combine accesses. On armcc, the way to signal this is done by accessing +// through the type (uint32 __packed *), but GCC has no such attribute +// (it ignores __attribute__((packed)) on individual variables). However, +// we can tell it that a _struct_ is unaligned, which has the same effect, +// so we do that. + +#elif defined(__arm__) && \ + !defined(__ARM_ARCH_4__) && \ + !defined(__ARM_ARCH_4T__) && \ + !defined(__ARM_ARCH_5__) && \ + !defined(__ARM_ARCH_5T__) && \ + !defined(__ARM_ARCH_5TE__) && \ + !defined(__ARM_ARCH_5TEJ__) && \ + !defined(__ARM_ARCH_6__) && \ + !defined(__ARM_ARCH_6J__) && \ + !defined(__ARM_ARCH_6K__) && \ + !defined(__ARM_ARCH_6Z__) && \ + !defined(__ARM_ARCH_6ZK__) && \ + !defined(__ARM_ARCH_6T2__) + +#if __GNUC__ +#define ATTRIBUTE_PACKED __attribute__((__packed__)) +#else +#define ATTRIBUTE_PACKED +#endif + +namespace base { +namespace internal { + +struct Unaligned16Struct { + uint16 value; + uint8 dummy; // To make the size non-power-of-two. +} ATTRIBUTE_PACKED; + +struct Unaligned32Struct { + uint32 value; + uint8 dummy; // To make the size non-power-of-two. +} ATTRIBUTE_PACKED; + +} // namespace internal +} // namespace base + +#define UNALIGNED_LOAD16(_p) \ + ((reinterpret_cast(_p))->value) +#define UNALIGNED_LOAD32(_p) \ + ((reinterpret_cast(_p))->value) + +#define UNALIGNED_STORE16(_p, _val) \ + ((reinterpret_cast< ::snappy::base::internal::Unaligned16Struct *>(_p))->value = \ + (_val)) +#define UNALIGNED_STORE32(_p, _val) \ + ((reinterpret_cast< ::snappy::base::internal::Unaligned32Struct *>(_p))->value = \ + (_val)) + +// TODO: NEON supports unaligned 64-bit loads and stores. +// See if that would be more efficient on platforms supporting it, +// at least for copies. + +inline uint64 UNALIGNED_LOAD64(const void *p) { + uint64 t; + memcpy(&t, p, sizeof t); + return t; +} + +inline void UNALIGNED_STORE64(void *p, uint64 v) { + memcpy(p, &v, sizeof v); +} + +#else + +// These functions are provided for architectures that don't support +// unaligned loads and stores. + +inline uint16 UNALIGNED_LOAD16(const void *p) { + uint16 t; + memcpy(&t, p, sizeof t); + return t; +} + +inline uint32 UNALIGNED_LOAD32(const void *p) { + uint32 t; + memcpy(&t, p, sizeof t); + return t; +} + +inline uint64 UNALIGNED_LOAD64(const void *p) { + uint64 t; + memcpy(&t, p, sizeof t); + return t; +} + +inline void UNALIGNED_STORE16(void *p, uint16 v) { + memcpy(p, &v, sizeof v); +} + +inline void UNALIGNED_STORE32(void *p, uint32 v) { + memcpy(p, &v, sizeof v); +} + +inline void UNALIGNED_STORE64(void *p, uint64 v) { + memcpy(p, &v, sizeof v); +} + +#endif + +// The following guarantees declaration of the byte swap functions. +#if defined(SNAPPY_IS_BIG_ENDIAN) + +#ifdef HAVE_SYS_BYTEORDER_H +#include +#endif + +#ifdef HAVE_SYS_ENDIAN_H +#include +#endif + +#ifdef _MSC_VER +#include +#define bswap_16(x) _byteswap_ushort(x) +#define bswap_32(x) _byteswap_ulong(x) +#define bswap_64(x) _byteswap_uint64(x) + +#elif defined(__APPLE__) +// Mac OS X / Darwin features +#include +#define bswap_16(x) OSSwapInt16(x) +#define bswap_32(x) OSSwapInt32(x) +#define bswap_64(x) OSSwapInt64(x) + +#elif defined(HAVE_BYTESWAP_H) +#include + +#elif defined(bswap32) +// FreeBSD defines bswap{16,32,64} in (already #included). +#define bswap_16(x) bswap16(x) +#define bswap_32(x) bswap32(x) +#define bswap_64(x) bswap64(x) + +#elif defined(BSWAP_64) +// Solaris 10 defines BSWAP_{16,32,64} in (already #included). +#define bswap_16(x) BSWAP_16(x) +#define bswap_32(x) BSWAP_32(x) +#define bswap_64(x) BSWAP_64(x) + +#else + +inline uint16 bswap_16(uint16 x) { + return (x << 8) | (x >> 8); +} + +inline uint32 bswap_32(uint32 x) { + x = ((x & 0xff00ff00UL) >> 8) | ((x & 0x00ff00ffUL) << 8); + return (x >> 16) | (x << 16); +} + +inline uint64 bswap_64(uint64 x) { + x = ((x & 0xff00ff00ff00ff00ULL) >> 8) | ((x & 0x00ff00ff00ff00ffULL) << 8); + x = ((x & 0xffff0000ffff0000ULL) >> 16) | ((x & 0x0000ffff0000ffffULL) << 16); + return (x >> 32) | (x << 32); +} + +#endif + +#endif // defined(SNAPPY_IS_BIG_ENDIAN) + +// Convert to little-endian storage, opposite of network format. +// Convert x from host to little endian: x = LittleEndian.FromHost(x); +// convert x from little endian to host: x = LittleEndian.ToHost(x); +// +// Store values into unaligned memory converting to little endian order: +// LittleEndian.Store16(p, x); +// +// Load unaligned values stored in little endian converting to host order: +// x = LittleEndian.Load16(p); +class LittleEndian { + public: + // Conversion functions. +#if defined(SNAPPY_IS_BIG_ENDIAN) + + static uint16 FromHost16(uint16 x) { return bswap_16(x); } + static uint16 ToHost16(uint16 x) { return bswap_16(x); } + + static uint32 FromHost32(uint32 x) { return bswap_32(x); } + static uint32 ToHost32(uint32 x) { return bswap_32(x); } + + static bool IsLittleEndian() { return false; } + +#else // !defined(SNAPPY_IS_BIG_ENDIAN) + + static uint16 FromHost16(uint16 x) { return x; } + static uint16 ToHost16(uint16 x) { return x; } + + static uint32 FromHost32(uint32 x) { return x; } + static uint32 ToHost32(uint32 x) { return x; } + + static bool IsLittleEndian() { return true; } + +#endif // !defined(SNAPPY_IS_BIG_ENDIAN) + + // Functions to do unaligned loads and stores in little-endian order. + static uint16 Load16(const void *p) { + return ToHost16(UNALIGNED_LOAD16(p)); + } + + static void Store16(void *p, uint16 v) { + UNALIGNED_STORE16(p, FromHost16(v)); + } + + static uint32 Load32(const void *p) { + return ToHost32(UNALIGNED_LOAD32(p)); + } + + static void Store32(void *p, uint32 v) { + UNALIGNED_STORE32(p, FromHost32(v)); + } +}; + +// Some bit-manipulation functions. +class Bits { + public: + // Return floor(log2(n)) for positive integer n. + static int Log2FloorNonZero(uint32 n); + + // Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0. + static int Log2Floor(uint32 n); + + // Return the first set least / most significant bit, 0-indexed. Returns an + // undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except + // that it's 0-indexed. + static int FindLSBSetNonZero(uint32 n); + +#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM) + static int FindLSBSetNonZero64(uint64 n); +#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM) + + private: + // No copying + Bits(const Bits&); + void operator=(const Bits&); +}; + +#ifdef HAVE_BUILTIN_CTZ + +inline int Bits::Log2FloorNonZero(uint32 n) { + assert(n != 0); + // (31 ^ x) is equivalent to (31 - x) for x in [0, 31]. An easy proof + // represents subtraction in base 2 and observes that there's no carry. + // + // GCC and Clang represent __builtin_clz on x86 as 31 ^ _bit_scan_reverse(x). + // Using "31 ^" here instead of "31 -" allows the optimizer to strip the + // function body down to _bit_scan_reverse(x). + return 31 ^ __builtin_clz(n); +} + +inline int Bits::Log2Floor(uint32 n) { + return (n == 0) ? -1 : Bits::Log2FloorNonZero(n); +} + +inline int Bits::FindLSBSetNonZero(uint32 n) { + assert(n != 0); + return __builtin_ctz(n); +} + +#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM) +inline int Bits::FindLSBSetNonZero64(uint64 n) { + assert(n != 0); + return __builtin_ctzll(n); +} +#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM) + +#elif defined(_MSC_VER) + +inline int Bits::Log2FloorNonZero(uint32 n) { + assert(n != 0); + unsigned long where; + _BitScanReverse(&where, n); + return static_cast(where); +} + +inline int Bits::Log2Floor(uint32 n) { + unsigned long where; + if (_BitScanReverse(&where, n)) + return static_cast(where); + return -1; +} + +inline int Bits::FindLSBSetNonZero(uint32 n) { + assert(n != 0); + unsigned long where; + if (_BitScanForward(&where, n)) + return static_cast(where); + return 32; +} + +#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM) +inline int Bits::FindLSBSetNonZero64(uint64 n) { + assert(n != 0); + unsigned long where; + if (_BitScanForward64(&where, n)) + return static_cast(where); + return 64; +} +#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM) + +#else // Portable versions. + +inline int Bits::Log2FloorNonZero(uint32 n) { + assert(n != 0); + + int log = 0; + uint32 value = n; + for (int i = 4; i >= 0; --i) { + int shift = (1 << i); + uint32 x = value >> shift; + if (x != 0) { + value = x; + log += shift; + } + } + assert(value == 1); + return log; +} + +inline int Bits::Log2Floor(uint32 n) { + return (n == 0) ? -1 : Bits::Log2FloorNonZero(n); +} + +inline int Bits::FindLSBSetNonZero(uint32 n) { + assert(n != 0); + + int rc = 31; + for (int i = 4, shift = 1 << 4; i >= 0; --i) { + const uint32 x = n << shift; + if (x != 0) { + n = x; + rc -= shift; + } + shift >>= 1; + } + return rc; +} + +#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM) +// FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero(). +inline int Bits::FindLSBSetNonZero64(uint64 n) { + assert(n != 0); + + const uint32 bottombits = static_cast(n); + if (bottombits == 0) { + // Bottom bits are zero, so scan in top bits + return 32 + FindLSBSetNonZero(static_cast(n >> 32)); + } else { + return FindLSBSetNonZero(bottombits); + } +} +#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM) + +#endif // End portable versions. + +// Variable-length integer encoding. +class Varint { + public: + // Maximum lengths of varint encoding of uint32. + static const int kMax32 = 5; + + // Attempts to parse a varint32 from a prefix of the bytes in [ptr,limit-1]. + // Never reads a character at or beyond limit. If a valid/terminated varint32 + // was found in the range, stores it in *OUTPUT and returns a pointer just + // past the last byte of the varint32. Else returns NULL. On success, + // "result <= limit". + static const char* Parse32WithLimit(const char* ptr, const char* limit, + uint32* OUTPUT); + + // REQUIRES "ptr" points to a buffer of length sufficient to hold "v". + // EFFECTS Encodes "v" into "ptr" and returns a pointer to the + // byte just past the last encoded byte. + static char* Encode32(char* ptr, uint32 v); + + // EFFECTS Appends the varint representation of "value" to "*s". + static void Append32(std::string* s, uint32 value); +}; + +inline const char* Varint::Parse32WithLimit(const char* p, + const char* l, + uint32* OUTPUT) { + const unsigned char* ptr = reinterpret_cast(p); + const unsigned char* limit = reinterpret_cast(l); + uint32 b, result; + if (ptr >= limit) return NULL; + b = *(ptr++); result = b & 127; if (b < 128) goto done; + if (ptr >= limit) return NULL; + b = *(ptr++); result |= (b & 127) << 7; if (b < 128) goto done; + if (ptr >= limit) return NULL; + b = *(ptr++); result |= (b & 127) << 14; if (b < 128) goto done; + if (ptr >= limit) return NULL; + b = *(ptr++); result |= (b & 127) << 21; if (b < 128) goto done; + if (ptr >= limit) return NULL; + b = *(ptr++); result |= (b & 127) << 28; if (b < 16) goto done; + return NULL; // Value is too long to be a varint32 + done: + *OUTPUT = result; + return reinterpret_cast(ptr); +} + +inline char* Varint::Encode32(char* sptr, uint32 v) { + // Operate on characters as unsigneds + unsigned char* ptr = reinterpret_cast(sptr); + static const int B = 128; + if (v < (1<<7)) { + *(ptr++) = v; + } else if (v < (1<<14)) { + *(ptr++) = v | B; + *(ptr++) = v>>7; + } else if (v < (1<<21)) { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = v>>14; + } else if (v < (1<<28)) { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = (v>>14) | B; + *(ptr++) = v>>21; + } else { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = (v>>14) | B; + *(ptr++) = (v>>21) | B; + *(ptr++) = v>>28; + } + return reinterpret_cast(ptr); +} + +// If you know the internal layout of the std::string in use, you can +// replace this function with one that resizes the string without +// filling the new space with zeros (if applicable) -- +// it will be non-portable but faster. +inline void STLStringResizeUninitialized(std::string* s, size_t new_size) { + s->resize(new_size); +} + +// Return a mutable char* pointing to a string's internal buffer, +// which may not be null-terminated. Writing through this pointer will +// modify the string. +// +// string_as_array(&str)[i] is valid for 0 <= i < str.size() until the +// next call to a string method that invalidates iterators. +// +// As of 2006-04, there is no standard-blessed way of getting a +// mutable reference to a string's internal buffer. However, issue 530 +// (http://www.open-std.org/JTC1/SC22/WG21/docs/lwg-defects.html#530) +// proposes this as the method. It will officially be part of the standard +// for C++0x. This should already work on all current implementations. +inline char* string_as_array(std::string* str) { + return str->empty() ? NULL : &*str->begin(); +} + +} // namespace snappy + +#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ diff --git a/target/snappy-1.1.10/snappy-stubs-public.h.in b/target/snappy-1.1.10/snappy-stubs-public.h.in new file mode 100644 index 00000000..416ab997 --- /dev/null +++ b/target/snappy-1.1.10/snappy-stubs-public.h.in @@ -0,0 +1,74 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Various type stubs for the open-source version of Snappy. +// +// This file cannot include config.h, as it is included from snappy.h, +// which is a public header. Instead, snappy-stubs-public.h is generated by +// from snappy-stubs-public.h.in at configure time. + +#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ +#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ + +#include +#include +#include + +#if ${HAVE_SYS_UIO_H_01} // HAVE_SYS_UIO_H +#include +#endif // HAVE_SYS_UIO_H + +#define SNAPPY_MAJOR ${PROJECT_VERSION_MAJOR} +#define SNAPPY_MINOR ${PROJECT_VERSION_MINOR} +#define SNAPPY_PATCHLEVEL ${PROJECT_VERSION_PATCH} +#define SNAPPY_VERSION \ + ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL) + +namespace snappy { + +using int8 = std::int8_t; +using uint8 = std::uint8_t; +using int16 = std::int16_t; +using uint16 = std::uint16_t; +using int32 = std::int32_t; +using uint32 = std::uint32_t; +using int64 = std::int64_t; +using uint64 = std::uint64_t; + +#if !${HAVE_SYS_UIO_H_01} // !HAVE_SYS_UIO_H +// Windows does not have an iovec type, yet the concept is universally useful. +// It is simple to define it ourselves, so we put it inside our own namespace. +struct iovec { + void* iov_base; + size_t iov_len; +}; +#endif // !HAVE_SYS_UIO_H + +} // namespace snappy + +#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ diff --git a/target/snappy-1.1.10/snappy-test.cc b/target/snappy-1.1.10/snappy-test.cc new file mode 100644 index 00000000..83be2d36 --- /dev/null +++ b/target/snappy-1.1.10/snappy-test.cc @@ -0,0 +1,613 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Various stubs for the unit tests for the open-source version of Snappy. + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_WINDOWS_H +// Needed to be able to use std::max without workarounds in the source code. +// https://support.microsoft.com/en-us/help/143208/prb-using-stl-in-windows-program-can-cause-min-max-conflicts +#define NOMINMAX +#include +#endif + +#include "snappy-test.h" + +#include + +DEFINE_bool(run_microbenchmarks, true, + "Run microbenchmarks before doing anything else."); + +namespace snappy { + +std::string ReadTestDataFile(const std::string& base, size_t size_limit) { + std::string contents; + const char* srcdir = getenv("srcdir"); // This is set by Automake. + std::string prefix; + if (srcdir) { + prefix = std::string(srcdir) + "/"; + } + file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults() + ).CheckSuccess(); + if (size_limit > 0) { + contents = contents.substr(0, size_limit); + } + return contents; +} + +std::string ReadTestDataFile(const std::string& base) { + return ReadTestDataFile(base, 0); +} + +std::string StrFormat(const char* format, ...) { + char buf[4096]; + va_list ap; + va_start(ap, format); + vsnprintf(buf, sizeof(buf), format, ap); + va_end(ap); + return buf; +} + +bool benchmark_running = false; +int64 benchmark_real_time_us = 0; +int64 benchmark_cpu_time_us = 0; +std::string* benchmark_label = nullptr; +int64 benchmark_bytes_processed = 0; + +void ResetBenchmarkTiming() { + benchmark_real_time_us = 0; + benchmark_cpu_time_us = 0; +} + +#ifdef WIN32 +LARGE_INTEGER benchmark_start_real; +FILETIME benchmark_start_cpu; +#else // WIN32 +struct timeval benchmark_start_real; +struct rusage benchmark_start_cpu; +#endif // WIN32 + +void StartBenchmarkTiming() { +#ifdef WIN32 + QueryPerformanceCounter(&benchmark_start_real); + FILETIME dummy; + CHECK(GetProcessTimes( + GetCurrentProcess(), &dummy, &dummy, &dummy, &benchmark_start_cpu)); +#else + gettimeofday(&benchmark_start_real, NULL); + if (getrusage(RUSAGE_SELF, &benchmark_start_cpu) == -1) { + perror("getrusage(RUSAGE_SELF)"); + exit(1); + } +#endif + benchmark_running = true; +} + +void StopBenchmarkTiming() { + if (!benchmark_running) { + return; + } + +#ifdef WIN32 + LARGE_INTEGER benchmark_stop_real; + LARGE_INTEGER benchmark_frequency; + QueryPerformanceCounter(&benchmark_stop_real); + QueryPerformanceFrequency(&benchmark_frequency); + + double elapsed_real = static_cast( + benchmark_stop_real.QuadPart - benchmark_start_real.QuadPart) / + benchmark_frequency.QuadPart; + benchmark_real_time_us += elapsed_real * 1e6 + 0.5; + + FILETIME benchmark_stop_cpu, dummy; + CHECK(GetProcessTimes( + GetCurrentProcess(), &dummy, &dummy, &dummy, &benchmark_stop_cpu)); + + ULARGE_INTEGER start_ulargeint; + start_ulargeint.LowPart = benchmark_start_cpu.dwLowDateTime; + start_ulargeint.HighPart = benchmark_start_cpu.dwHighDateTime; + + ULARGE_INTEGER stop_ulargeint; + stop_ulargeint.LowPart = benchmark_stop_cpu.dwLowDateTime; + stop_ulargeint.HighPart = benchmark_stop_cpu.dwHighDateTime; + + benchmark_cpu_time_us += + (stop_ulargeint.QuadPart - start_ulargeint.QuadPart + 5) / 10; +#else // WIN32 + struct timeval benchmark_stop_real; + gettimeofday(&benchmark_stop_real, NULL); + benchmark_real_time_us += + 1000000 * (benchmark_stop_real.tv_sec - benchmark_start_real.tv_sec); + benchmark_real_time_us += + (benchmark_stop_real.tv_usec - benchmark_start_real.tv_usec); + + struct rusage benchmark_stop_cpu; + if (getrusage(RUSAGE_SELF, &benchmark_stop_cpu) == -1) { + perror("getrusage(RUSAGE_SELF)"); + exit(1); + } + benchmark_cpu_time_us += 1000000 * (benchmark_stop_cpu.ru_utime.tv_sec - + benchmark_start_cpu.ru_utime.tv_sec); + benchmark_cpu_time_us += (benchmark_stop_cpu.ru_utime.tv_usec - + benchmark_start_cpu.ru_utime.tv_usec); +#endif // WIN32 + + benchmark_running = false; +} + +void SetBenchmarkLabel(const std::string& str) { + if (benchmark_label) { + delete benchmark_label; + } + benchmark_label = new std::string(str); +} + +void SetBenchmarkBytesProcessed(int64 bytes) { + benchmark_bytes_processed = bytes; +} + +struct BenchmarkRun { + int64 real_time_us; + int64 cpu_time_us; +}; + +struct BenchmarkCompareCPUTime { + bool operator() (const BenchmarkRun& a, const BenchmarkRun& b) const { + return a.cpu_time_us < b.cpu_time_us; + } +}; + +void Benchmark::Run() { + for (int test_case_num = start_; test_case_num <= stop_; ++test_case_num) { + // Run a few iterations first to find out approximately how fast + // the benchmark is. + const int kCalibrateIterations = 100; + ResetBenchmarkTiming(); + StartBenchmarkTiming(); + (*function_)(kCalibrateIterations, test_case_num); + StopBenchmarkTiming(); + + // Let each test case run for about 200ms, but at least as many + // as we used to calibrate. + // Run five times and pick the median. + const int kNumRuns = 5; + const int kMedianPos = kNumRuns / 2; + int num_iterations = 0; + if (benchmark_real_time_us > 0) { + num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us; + } + num_iterations = std::max(num_iterations, kCalibrateIterations); + BenchmarkRun benchmark_runs[kNumRuns]; + + for (int run = 0; run < kNumRuns; ++run) { + ResetBenchmarkTiming(); + StartBenchmarkTiming(); + (*function_)(num_iterations, test_case_num); + StopBenchmarkTiming(); + + benchmark_runs[run].real_time_us = benchmark_real_time_us; + benchmark_runs[run].cpu_time_us = benchmark_cpu_time_us; + } + + std::string heading = StrFormat("%s/%d", name_.c_str(), test_case_num); + std::string human_readable_speed; + + std::nth_element(benchmark_runs, + benchmark_runs + kMedianPos, + benchmark_runs + kNumRuns, + BenchmarkCompareCPUTime()); + int64 real_time_us = benchmark_runs[kMedianPos].real_time_us; + int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us; + if (cpu_time_us <= 0) { + human_readable_speed = "?"; + } else { + int64 bytes_per_second = + benchmark_bytes_processed * 1000000 / cpu_time_us; + if (bytes_per_second < 1024) { + human_readable_speed = + StrFormat("%dB/s", static_cast(bytes_per_second)); + } else if (bytes_per_second < 1024 * 1024) { + human_readable_speed = StrFormat( + "%.1fkB/s", bytes_per_second / 1024.0f); + } else if (bytes_per_second < 1024 * 1024 * 1024) { + human_readable_speed = StrFormat( + "%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f)); + } else { + human_readable_speed = StrFormat( + "%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f)); + } + } + + fprintf(stderr, +#ifdef WIN32 + "%-18s %10I64d %10I64d %10d %s %s\n", +#else + "%-18s %10lld %10lld %10d %s %s\n", +#endif + heading.c_str(), + static_cast(real_time_us * 1000 / num_iterations), + static_cast(cpu_time_us * 1000 / num_iterations), + num_iterations, + human_readable_speed.c_str(), + benchmark_label->c_str()); + } +} + +#ifdef HAVE_LIBZ + +ZLib::ZLib() + : comp_init_(false), + uncomp_init_(false) { + Reinit(); +} + +ZLib::~ZLib() { + if (comp_init_) { deflateEnd(&comp_stream_); } + if (uncomp_init_) { inflateEnd(&uncomp_stream_); } +} + +void ZLib::Reinit() { + compression_level_ = Z_DEFAULT_COMPRESSION; + window_bits_ = MAX_WBITS; + mem_level_ = 8; // DEF_MEM_LEVEL + if (comp_init_) { + deflateEnd(&comp_stream_); + comp_init_ = false; + } + if (uncomp_init_) { + inflateEnd(&uncomp_stream_); + uncomp_init_ = false; + } + first_chunk_ = true; +} + +void ZLib::Reset() { + first_chunk_ = true; +} + +// --------- COMPRESS MODE + +// Initialization method to be called if we hit an error while +// compressing. On hitting an error, call this method before returning +// the error. +void ZLib::CompressErrorInit() { + deflateEnd(&comp_stream_); + comp_init_ = false; + Reset(); +} + +int ZLib::DeflateInit() { + return deflateInit2(&comp_stream_, + compression_level_, + Z_DEFLATED, + window_bits_, + mem_level_, + Z_DEFAULT_STRATEGY); +} + +int ZLib::CompressInit(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen) { + int err; + + comp_stream_.next_in = (Bytef*)source; + comp_stream_.avail_in = (uInt)*sourceLen; + if ((uLong)comp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR; + comp_stream_.next_out = dest; + comp_stream_.avail_out = (uInt)*destLen; + if ((uLong)comp_stream_.avail_out != *destLen) return Z_BUF_ERROR; + + if ( !first_chunk_ ) // only need to set up stream the first time through + return Z_OK; + + if (comp_init_) { // we've already initted it + err = deflateReset(&comp_stream_); + if (err != Z_OK) { + LOG(WARNING) << "ERROR: Can't reset compress object; creating a new one"; + deflateEnd(&comp_stream_); + comp_init_ = false; + } + } + if (!comp_init_) { // first use + comp_stream_.zalloc = (alloc_func)0; + comp_stream_.zfree = (free_func)0; + comp_stream_.opaque = (voidpf)0; + err = DeflateInit(); + if (err != Z_OK) return err; + comp_init_ = true; + } + return Z_OK; +} + +// In a perfect world we'd always have the full buffer to compress +// when the time came, and we could just call Compress(). Alas, we +// want to do chunked compression on our webserver. In this +// application, we compress the header, send it off, then compress the +// results, send them off, then compress the footer. Thus we need to +// use the chunked compression features of zlib. +int ZLib::CompressAtMostOrAll(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen, + int flush_mode) { // Z_FULL_FLUSH or Z_FINISH + int err; + + if ( (err=CompressInit(dest, destLen, source, sourceLen)) != Z_OK ) + return err; + + // This is used to figure out how many bytes we wrote *this chunk* + int compressed_size = comp_stream_.total_out; + + // Some setup happens only for the first chunk we compress in a run + if ( first_chunk_ ) { + first_chunk_ = false; + } + + // flush_mode is Z_FINISH for all mode, Z_SYNC_FLUSH for incremental + // compression. + err = deflate(&comp_stream_, flush_mode); + + *sourceLen = comp_stream_.avail_in; + + if ((err == Z_STREAM_END || err == Z_OK) + && comp_stream_.avail_in == 0 + && comp_stream_.avail_out != 0 ) { + // we processed everything ok and the output buffer was large enough. + ; + } else if (err == Z_STREAM_END && comp_stream_.avail_in > 0) { + return Z_BUF_ERROR; // should never happen + } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) { + // an error happened + CompressErrorInit(); + return err; + } else if (comp_stream_.avail_out == 0) { // not enough space + err = Z_BUF_ERROR; + } + + assert(err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR); + if (err == Z_STREAM_END) + err = Z_OK; + + // update the crc and other metadata + compressed_size = comp_stream_.total_out - compressed_size; // delta + *destLen = compressed_size; + + return err; +} + +int ZLib::CompressChunkOrAll(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int flush_mode) { // Z_FULL_FLUSH or Z_FINISH + const int ret = + CompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode); + if (ret == Z_BUF_ERROR) + CompressErrorInit(); + return ret; +} + +// This routine only initializes the compression stream once. Thereafter, it +// just does a deflateReset on the stream, which should be faster. +int ZLib::Compress(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen) { + int err; + if ( (err=CompressChunkOrAll(dest, destLen, source, sourceLen, + Z_FINISH)) != Z_OK ) + return err; + Reset(); // reset for next call to Compress + + return Z_OK; +} + + +// --------- UNCOMPRESS MODE + +int ZLib::InflateInit() { + return inflateInit2(&uncomp_stream_, MAX_WBITS); +} + +// Initialization method to be called if we hit an error while +// uncompressing. On hitting an error, call this method before +// returning the error. +void ZLib::UncompressErrorInit() { + inflateEnd(&uncomp_stream_); + uncomp_init_ = false; + Reset(); +} + +int ZLib::UncompressInit(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen) { + int err; + + uncomp_stream_.next_in = (Bytef*)source; + uncomp_stream_.avail_in = (uInt)*sourceLen; + // Check for source > 64K on 16-bit machine: + if ((uLong)uncomp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR; + + uncomp_stream_.next_out = dest; + uncomp_stream_.avail_out = (uInt)*destLen; + if ((uLong)uncomp_stream_.avail_out != *destLen) return Z_BUF_ERROR; + + if ( !first_chunk_ ) // only need to set up stream the first time through + return Z_OK; + + if (uncomp_init_) { // we've already initted it + err = inflateReset(&uncomp_stream_); + if (err != Z_OK) { + LOG(WARNING) + << "ERROR: Can't reset uncompress object; creating a new one"; + UncompressErrorInit(); + } + } + if (!uncomp_init_) { + uncomp_stream_.zalloc = (alloc_func)0; + uncomp_stream_.zfree = (free_func)0; + uncomp_stream_.opaque = (voidpf)0; + err = InflateInit(); + if (err != Z_OK) return err; + uncomp_init_ = true; + } + return Z_OK; +} + +// If you compressed your data a chunk at a time, with CompressChunk, +// you can uncompress it a chunk at a time with UncompressChunk. +// Only difference bewteen chunked and unchunked uncompression +// is the flush mode we use: Z_SYNC_FLUSH (chunked) or Z_FINISH (unchunked). +int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen, + int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH + int err = Z_OK; + + if ( (err=UncompressInit(dest, destLen, source, sourceLen)) != Z_OK ) { + LOG(WARNING) << "UncompressInit: Error: " << err << " SourceLen: " + << *sourceLen; + return err; + } + + // This is used to figure out how many output bytes we wrote *this chunk*: + const uLong old_total_out = uncomp_stream_.total_out; + + // This is used to figure out how many input bytes we read *this chunk*: + const uLong old_total_in = uncomp_stream_.total_in; + + // Some setup happens only for the first chunk we compress in a run + if ( first_chunk_ ) { + first_chunk_ = false; // so we don't do this again + + // For the first chunk *only* (to avoid infinite troubles), we let + // there be no actual data to uncompress. This sometimes triggers + // when the input is only the gzip header, say. + if ( *sourceLen == 0 ) { + *destLen = 0; + return Z_OK; + } + } + + // We'll uncompress as much as we can. If we end OK great, otherwise + // if we get an error that seems to be the gzip footer, we store the + // gzip footer and return OK, otherwise we return the error. + + // flush_mode is Z_SYNC_FLUSH for chunked mode, Z_FINISH for all mode. + err = inflate(&uncomp_stream_, flush_mode); + + // Figure out how many bytes of the input zlib slurped up: + const uLong bytes_read = uncomp_stream_.total_in - old_total_in; + CHECK_LE(source + bytes_read, source + *sourceLen); + *sourceLen = uncomp_stream_.avail_in; + + if ((err == Z_STREAM_END || err == Z_OK) // everything went ok + && uncomp_stream_.avail_in == 0) { // and we read it all + ; + } else if (err == Z_STREAM_END && uncomp_stream_.avail_in > 0) { + LOG(WARNING) + << "UncompressChunkOrAll: Received some extra data, bytes total: " + << uncomp_stream_.avail_in << " bytes: " + << std::string(reinterpret_cast(uncomp_stream_.next_in), + std::min(int(uncomp_stream_.avail_in), 20)); + UncompressErrorInit(); + return Z_DATA_ERROR; // what's the extra data for? + } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) { + // an error happened + LOG(WARNING) << "UncompressChunkOrAll: Error: " << err + << " avail_out: " << uncomp_stream_.avail_out; + UncompressErrorInit(); + return err; + } else if (uncomp_stream_.avail_out == 0) { + err = Z_BUF_ERROR; + } + + assert(err == Z_OK || err == Z_BUF_ERROR || err == Z_STREAM_END); + if (err == Z_STREAM_END) + err = Z_OK; + + *destLen = uncomp_stream_.total_out - old_total_out; // size for this call + + return err; +} + +int ZLib::UncompressChunkOrAll(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH + const int ret = + UncompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode); + if (ret == Z_BUF_ERROR) + UncompressErrorInit(); + return ret; +} + +int ZLib::UncompressAtMost(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen) { + return UncompressAtMostOrAll(dest, destLen, source, sourceLen, Z_SYNC_FLUSH); +} + +// We make sure we've uncompressed everything, that is, the current +// uncompress stream is at a compressed-buffer-EOF boundary. In gzip +// mode, we also check the gzip footer to make sure we pass the gzip +// consistency checks. We RETURN true iff both types of checks pass. +bool ZLib::UncompressChunkDone() { + assert(!first_chunk_ && uncomp_init_); + // Make sure we're at the end-of-compressed-data point. This means + // if we call inflate with Z_FINISH we won't consume any input or + // write any output + Bytef dummyin, dummyout; + uLongf dummylen = 0; + if ( UncompressChunkOrAll(&dummyout, &dummylen, &dummyin, 0, Z_FINISH) + != Z_OK ) { + return false; + } + + // Make sure that when we exit, we can start a new round of chunks later + Reset(); + + return true; +} + +// Uncompresses the source buffer into the destination buffer. +// The destination buffer must be long enough to hold the entire +// decompressed contents. +// +// We only initialize the uncomp_stream once. Thereafter, we use +// inflateReset, which should be faster. +// +// Returns Z_OK on success, otherwise, it returns a zlib error code. +int ZLib::Uncompress(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen) { + int err; + if ( (err=UncompressChunkOrAll(dest, destLen, source, sourceLen, + Z_FINISH)) != Z_OK ) { + Reset(); // let us try to compress again + return err; + } + if ( !UncompressChunkDone() ) // calls Reset() + return Z_DATA_ERROR; + return Z_OK; // stream_end is ok +} + +#endif // HAVE_LIBZ + +} // namespace snappy diff --git a/target/snappy-1.1.10/snappy-test.h b/target/snappy-1.1.10/snappy-test.h new file mode 100644 index 00000000..c8b7d388 --- /dev/null +++ b/target/snappy-1.1.10/snappy-test.h @@ -0,0 +1,525 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Various stubs for the unit tests for the open-source version of Snappy. + +#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_ +#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_ + +#include +#include + +#include "snappy-stubs-internal.h" + +#include +#include + +#ifdef HAVE_SYS_MMAN_H +#include +#endif + +#ifdef HAVE_SYS_RESOURCE_H +#include +#endif + +#ifdef HAVE_SYS_TIME_H +#include +#endif + +#ifdef HAVE_WINDOWS_H +#include +#endif + +#ifdef HAVE_GTEST + +#include +#undef TYPED_TEST +#define TYPED_TEST TEST +#define INIT_GTEST(argc, argv) ::testing::InitGoogleTest(argc, *argv) + +#else + +// Stubs for if the user doesn't have Google Test installed. + +#define TEST(test_case, test_subcase) \ + void Test_ ## test_case ## _ ## test_subcase() +#define INIT_GTEST(argc, argv) + +#define TYPED_TEST TEST +#define EXPECT_EQ CHECK_EQ +#define EXPECT_NE CHECK_NE +#define EXPECT_FALSE(cond) CHECK(!(cond)) + +#endif + +#ifdef HAVE_GFLAGS + +#include + +// This is tricky; both gflags and Google Test want to look at the command line +// arguments. Google Test seems to be the most happy with unknown arguments, +// though, so we call it first and hope for the best. +#define InitGoogle(argv0, argc, argv, remove_flags) \ + INIT_GTEST(argc, argv); \ + google::ParseCommandLineFlags(argc, argv, remove_flags); + +#else + +// If we don't have the gflags package installed, these can only be +// changed at compile time. +#define DEFINE_int32(flag_name, default_value, description) \ + static int FLAGS_ ## flag_name = default_value; + +#define InitGoogle(argv0, argc, argv, remove_flags) \ + INIT_GTEST(argc, argv) + +#endif + +#ifdef HAVE_LIBZ +#include "zlib.h" +#endif + +#ifdef HAVE_LIBLZO2 +#include "lzo/lzo1x.h" +#endif + +namespace { + +namespace file { + int Defaults() { return 0; } + + class DummyStatus { + public: + void CheckSuccess() { } + }; + + DummyStatus GetContents( + const std::string& filename, std::string* data, int unused) { + FILE* fp = fopen(filename.c_str(), "rb"); + if (fp == NULL) { + perror(filename.c_str()); + exit(1); + } + + data->clear(); + while (!feof(fp)) { + char buf[4096]; + size_t ret = fread(buf, 1, 4096, fp); + if (ret == 0 && ferror(fp)) { + perror("fread"); + exit(1); + } + data->append(std::string(buf, ret)); + } + + fclose(fp); + + return DummyStatus(); + } + + inline DummyStatus SetContents( + const std::string& filename, const std::string& str, int unused) { + FILE* fp = fopen(filename.c_str(), "wb"); + if (fp == NULL) { + perror(filename.c_str()); + exit(1); + } + + int ret = fwrite(str.data(), str.size(), 1, fp); + if (ret != 1) { + perror("fwrite"); + exit(1); + } + + fclose(fp); + + return DummyStatus(); + } +} // namespace file + +} // namespace + +namespace snappy { + +#define FLAGS_test_random_seed 301 +using TypeParam = std::string; + +void Test_CorruptedTest_VerifyCorrupted(); +void Test_Snappy_SimpleTests(); +void Test_Snappy_MaxBlowup(); +void Test_Snappy_RandomData(); +void Test_Snappy_FourByteOffset(); +void Test_SnappyCorruption_TruncatedVarint(); +void Test_SnappyCorruption_UnterminatedVarint(); +void Test_SnappyCorruption_OverflowingVarint(); +void Test_Snappy_ReadPastEndOfBuffer(); +void Test_Snappy_FindMatchLength(); +void Test_Snappy_FindMatchLengthRandom(); + +std::string ReadTestDataFile(const std::string& base, size_t size_limit); + +std::string ReadTestDataFile(const std::string& base); + +// A sprintf() variant that returns a std::string. +// Not safe for general use due to truncation issues. +std::string StrFormat(const char* format, ...); + +// A wall-time clock. This stub is not super-accurate, nor resistant to the +// system time changing. +class CycleTimer { + public: + CycleTimer() : real_time_us_(0) {} + + void Start() { +#ifdef WIN32 + QueryPerformanceCounter(&start_); +#else + gettimeofday(&start_, NULL); +#endif + } + + void Stop() { +#ifdef WIN32 + LARGE_INTEGER stop; + LARGE_INTEGER frequency; + QueryPerformanceCounter(&stop); + QueryPerformanceFrequency(&frequency); + + double elapsed = static_cast(stop.QuadPart - start_.QuadPart) / + frequency.QuadPart; + real_time_us_ += elapsed * 1e6 + 0.5; +#else + struct timeval stop; + gettimeofday(&stop, NULL); + + real_time_us_ += 1000000 * (stop.tv_sec - start_.tv_sec); + real_time_us_ += (stop.tv_usec - start_.tv_usec); +#endif + } + + double Get() { + return real_time_us_ * 1e-6; + } + + private: + int64 real_time_us_; +#ifdef WIN32 + LARGE_INTEGER start_; +#else + struct timeval start_; +#endif +}; + +// Minimalistic microbenchmark framework. + +typedef void (*BenchmarkFunction)(int, int); + +class Benchmark { + public: + Benchmark(const std::string& name, BenchmarkFunction function) + : name_(name), function_(function) {} + + Benchmark* DenseRange(int start, int stop) { + start_ = start; + stop_ = stop; + return this; + } + + void Run(); + + private: + const std::string name_; + const BenchmarkFunction function_; + int start_, stop_; +}; +#define BENCHMARK(benchmark_name) \ + Benchmark* Benchmark_ ## benchmark_name = \ + (new Benchmark(#benchmark_name, benchmark_name)) + +extern Benchmark* Benchmark_BM_UFlat; +extern Benchmark* Benchmark_BM_UIOVec; +extern Benchmark* Benchmark_BM_UValidate; +extern Benchmark* Benchmark_BM_ZFlat; +extern Benchmark* Benchmark_BM_ZFlatAll; +extern Benchmark* Benchmark_BM_ZFlatIncreasingTableSize; + +void ResetBenchmarkTiming(); +void StartBenchmarkTiming(); +void StopBenchmarkTiming(); +void SetBenchmarkLabel(const std::string& str); +void SetBenchmarkBytesProcessed(int64 bytes); + +#ifdef HAVE_LIBZ + +// Object-oriented wrapper around zlib. +class ZLib { + public: + ZLib(); + ~ZLib(); + + // Wipe a ZLib object to a virgin state. This differs from Reset() + // in that it also breaks any state. + void Reinit(); + + // Call this to make a zlib buffer as good as new. Here's the only + // case where they differ: + // CompressChunk(a); CompressChunk(b); CompressChunkDone(); vs + // CompressChunk(a); Reset(); CompressChunk(b); CompressChunkDone(); + // You'll want to use Reset(), then, when you interrupt a compress + // (or uncompress) in the middle of a chunk and want to start over. + void Reset(); + + // According to the zlib manual, when you Compress, the destination + // buffer must have size at least src + .1%*src + 12. This function + // helps you calculate that. Augment this to account for a potential + // gzip header and footer, plus a few bytes of slack. + static int MinCompressbufSize(int uncompress_size) { + return uncompress_size + uncompress_size/1000 + 40; + } + + // Compresses the source buffer into the destination buffer. + // sourceLen is the byte length of the source buffer. + // Upon entry, destLen is the total size of the destination buffer, + // which must be of size at least MinCompressbufSize(sourceLen). + // Upon exit, destLen is the actual size of the compressed buffer. + // + // This function can be used to compress a whole file at once if the + // input file is mmap'ed. + // + // Returns Z_OK if success, Z_MEM_ERROR if there was not + // enough memory, Z_BUF_ERROR if there was not enough room in the + // output buffer. Note that if the output buffer is exactly the same + // size as the compressed result, we still return Z_BUF_ERROR. + // (check CL#1936076) + int Compress(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen); + + // Uncompresses the source buffer into the destination buffer. + // The destination buffer must be long enough to hold the entire + // decompressed contents. + // + // Returns Z_OK on success, otherwise, it returns a zlib error code. + int Uncompress(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen); + + // Uncompress data one chunk at a time -- ie you can call this + // more than once. To get this to work you need to call per-chunk + // and "done" routines. + // + // Returns Z_OK if success, Z_MEM_ERROR if there was not + // enough memory, Z_BUF_ERROR if there was not enough room in the + // output buffer. + + int UncompressAtMost(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen); + + // Checks gzip footer information, as needed. Mostly this just + // makes sure the checksums match. Whenever you call this, it + // will assume the last 8 bytes from the previous UncompressChunk + // call are the footer. Returns true iff everything looks ok. + bool UncompressChunkDone(); + + private: + int InflateInit(); // sets up the zlib inflate structure + int DeflateInit(); // sets up the zlib deflate structure + + // These init the zlib data structures for compressing/uncompressing + int CompressInit(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen); + int UncompressInit(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen); + // Initialization method to be called if we hit an error while + // uncompressing. On hitting an error, call this method before + // returning the error. + void UncompressErrorInit(); + + // Helper function for Compress + int CompressChunkOrAll(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int flush_mode); + int CompressAtMostOrAll(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen, + int flush_mode); + + // Likewise for UncompressAndUncompressChunk + int UncompressChunkOrAll(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int flush_mode); + + int UncompressAtMostOrAll(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen, + int flush_mode); + + // Initialization method to be called if we hit an error while + // compressing. On hitting an error, call this method before + // returning the error. + void CompressErrorInit(); + + int compression_level_; // compression level + int window_bits_; // log base 2 of the window size used in compression + int mem_level_; // specifies the amount of memory to be used by + // compressor (1-9) + z_stream comp_stream_; // Zlib stream data structure + bool comp_init_; // True if we have initialized comp_stream_ + z_stream uncomp_stream_; // Zlib stream data structure + bool uncomp_init_; // True if we have initialized uncomp_stream_ + + // These are used only with chunked compression. + bool first_chunk_; // true if we need to emit headers with this chunk +}; + +#endif // HAVE_LIBZ + +} // namespace snappy + +DECLARE_bool(run_microbenchmarks); + +static inline void RunSpecifiedBenchmarks() { + if (!FLAGS_run_microbenchmarks) { + return; + } + + fprintf(stderr, "Running microbenchmarks.\n"); +#ifndef NDEBUG + fprintf(stderr, "WARNING: Compiled with assertions enabled, will be slow.\n"); +#endif +#ifndef __OPTIMIZE__ + fprintf(stderr, "WARNING: Compiled without optimization, will be slow.\n"); +#endif + fprintf(stderr, "Benchmark Time(ns) CPU(ns) Iterations\n"); + fprintf(stderr, "---------------------------------------------------\n"); + + snappy::Benchmark_BM_UFlat->Run(); + snappy::Benchmark_BM_UIOVec->Run(); + snappy::Benchmark_BM_UValidate->Run(); + snappy::Benchmark_BM_ZFlat->Run(); + snappy::Benchmark_BM_ZFlatAll->Run(); + snappy::Benchmark_BM_ZFlatIncreasingTableSize->Run(); + + fprintf(stderr, "\n"); +} + +#ifndef HAVE_GTEST + +static inline int RUN_ALL_TESTS() { + fprintf(stderr, "Running correctness tests.\n"); + snappy::Test_CorruptedTest_VerifyCorrupted(); + snappy::Test_Snappy_SimpleTests(); + snappy::Test_Snappy_MaxBlowup(); + snappy::Test_Snappy_RandomData(); + snappy::Test_Snappy_FourByteOffset(); + snappy::Test_SnappyCorruption_TruncatedVarint(); + snappy::Test_SnappyCorruption_UnterminatedVarint(); + snappy::Test_SnappyCorruption_OverflowingVarint(); + snappy::Test_Snappy_ReadPastEndOfBuffer(); + snappy::Test_Snappy_FindMatchLength(); + snappy::Test_Snappy_FindMatchLengthRandom(); + fprintf(stderr, "All tests passed.\n"); + + return 0; +} + +#endif // HAVE_GTEST + +// For main(). +namespace snappy { + +// Logging. + +#define LOG(level) LogMessage() +#define VLOG(level) true ? (void)0 : \ + snappy::LogMessageVoidify() & snappy::LogMessage() + +class LogMessage { + public: + LogMessage() { } + ~LogMessage() { + std::cerr << std::endl; + } + + LogMessage& operator<<(const std::string& msg) { + std::cerr << msg; + return *this; + } + LogMessage& operator<<(int x) { + std::cerr << x; + return *this; + } +}; + +// Asserts, both versions activated in debug mode only, +// and ones that are always active. + +#define CRASH_UNLESS(condition) \ + SNAPPY_PREDICT_TRUE(condition) ? (void)0 : \ + snappy::LogMessageVoidify() & snappy::LogMessageCrash() + +#ifdef _MSC_VER +// ~LogMessageCrash calls abort() and therefore never exits. This is by design +// so temporarily disable warning C4722. +#pragma warning(push) +#pragma warning(disable:4722) +#endif + +class LogMessageCrash : public LogMessage { + public: + LogMessageCrash() { } + ~LogMessageCrash() { + std::cerr << std::endl; + abort(); + } +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +// This class is used to explicitly ignore values in the conditional +// logging macros. This avoids compiler warnings like "value computed +// is not used" and "statement has no effect". + +class LogMessageVoidify { + public: + LogMessageVoidify() { } + // This has to be an operator with a precedence lower than << but + // higher than ?: + void operator&(const LogMessage&) { } +}; + +#define CHECK(cond) CRASH_UNLESS(cond) +#define CHECK_LE(a, b) CRASH_UNLESS((a) <= (b)) +#define CHECK_GE(a, b) CRASH_UNLESS((a) >= (b)) +#define CHECK_EQ(a, b) CRASH_UNLESS((a) == (b)) +#define CHECK_NE(a, b) CRASH_UNLESS((a) != (b)) +#define CHECK_LT(a, b) CRASH_UNLESS((a) < (b)) +#define CHECK_GT(a, b) CRASH_UNLESS((a) > (b)) +#define CHECK_OK(cond) (cond).CheckSuccess() + +} // namespace snappy + +#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_ diff --git a/target/snappy-1.1.10/snappy.cc b/target/snappy-1.1.10/snappy.cc new file mode 100644 index 00000000..ce1eef46 --- /dev/null +++ b/target/snappy-1.1.10/snappy.cc @@ -0,0 +1,1661 @@ +// Copyright 2005 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "snappy.h" +#include "snappy-internal.h" +#include "snappy-sinksource.h" + +#if !defined(SNAPPY_HAVE_SSSE3) +// __SSSE3__ is defined by GCC and Clang. Visual Studio doesn't target SIMD +// support between SSE2 and AVX (so SSSE3 instructions require AVX support), and +// defines __AVX__ when AVX support is available. +#if defined(__SSSE3__) || defined(__AVX__) +#define SNAPPY_HAVE_SSSE3 1 +#else +#define SNAPPY_HAVE_SSSE3 0 +#endif +#endif // !defined(SNAPPY_HAVE_SSSE3) + +#if !defined(SNAPPY_HAVE_BMI2) +// __BMI2__ is defined by GCC and Clang. Visual Studio doesn't target BMI2 +// specifically, but it does define __AVX2__ when AVX2 support is available. +// Fortunately, AVX2 was introduced in Haswell, just like BMI2. +// +// BMI2 is not defined as a subset of AVX2 (unlike SSSE3 and AVX above). So, +// GCC and Clang can build code with AVX2 enabled but BMI2 disabled, in which +// case issuing BMI2 instructions results in a compiler error. +#if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__)) +#define SNAPPY_HAVE_BMI2 1 +#else +#define SNAPPY_HAVE_BMI2 0 +#endif +#endif // !defined(SNAPPY_HAVE_BMI2) + +#if SNAPPY_HAVE_SSSE3 +// Please do not replace with . or with headers that assume more +// advanced SSE versions without checking with all the OWNERS. +#include +#endif + +#if SNAPPY_HAVE_BMI2 +// Please do not replace with . or with headers that assume more +// advanced SSE versions without checking with all the OWNERS. +#include +#endif + +#include + +#include +#include +#include + +namespace snappy { + +using internal::COPY_1_BYTE_OFFSET; +using internal::COPY_2_BYTE_OFFSET; +using internal::LITERAL; +using internal::char_table; +using internal::kMaximumTagLength; + +// Any hash function will produce a valid compressed bitstream, but a good +// hash function reduces the number of collisions and thus yields better +// compression for compressible input, and more speed for incompressible +// input. Of course, it doesn't hurt if the hash function is reasonably fast +// either, as it gets called a lot. +static inline uint32 HashBytes(uint32 bytes, int shift) { + uint32 kMul = 0x1e35a7bd; + return (bytes * kMul) >> shift; +} +static inline uint32 Hash(const char* p, int shift) { + return HashBytes(UNALIGNED_LOAD32(p), shift); +} + +size_t MaxCompressedLength(size_t source_len) { + // Compressed data can be defined as: + // compressed := item* literal* + // item := literal* copy + // + // The trailing literal sequence has a space blowup of at most 62/60 + // since a literal of length 60 needs one tag byte + one extra byte + // for length information. + // + // Item blowup is trickier to measure. Suppose the "copy" op copies + // 4 bytes of data. Because of a special check in the encoding code, + // we produce a 4-byte copy only if the offset is < 65536. Therefore + // the copy op takes 3 bytes to encode, and this type of item leads + // to at most the 62/60 blowup for representing literals. + // + // Suppose the "copy" op copies 5 bytes of data. If the offset is big + // enough, it will take 5 bytes to encode the copy op. Therefore the + // worst case here is a one-byte literal followed by a five-byte copy. + // I.e., 6 bytes of input turn into 7 bytes of "compressed" data. + // + // This last factor dominates the blowup, so the final estimate is: + return 32 + source_len + source_len/6; +} + +namespace { + +void UnalignedCopy64(const void* src, void* dst) { + char tmp[8]; + memcpy(tmp, src, 8); + memcpy(dst, tmp, 8); +} + +void UnalignedCopy128(const void* src, void* dst) { + // memcpy gets vectorized when the appropriate compiler options are used. + // For example, x86 compilers targeting SSE2+ will optimize to an SSE2 load + // and store. + char tmp[16]; + memcpy(tmp, src, 16); + memcpy(dst, tmp, 16); +} + +// Copy [src, src+(op_limit-op)) to [op, (op_limit-op)) a byte at a time. Used +// for handling COPY operations where the input and output regions may overlap. +// For example, suppose: +// src == "ab" +// op == src + 2 +// op_limit == op + 20 +// After IncrementalCopySlow(src, op, op_limit), the result will have eleven +// copies of "ab" +// ababababababababababab +// Note that this does not match the semantics of either memcpy() or memmove(). +inline char* IncrementalCopySlow(const char* src, char* op, + char* const op_limit) { + // TODO: Remove pragma when LLVM is aware this + // function is only called in cold regions and when cold regions don't get + // vectorized or unrolled. +#ifdef __clang__ +#pragma clang loop unroll(disable) +#endif + while (op < op_limit) { + *op++ = *src++; + } + return op_limit; +} + +#if SNAPPY_HAVE_SSSE3 + +// This is a table of shuffle control masks that can be used as the source +// operand for PSHUFB to permute the contents of the destination XMM register +// into a repeating byte pattern. +alignas(16) const char pshufb_fill_patterns[7][16] = { + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}, + {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0}, + {0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3}, + {0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0}, + {0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3}, + {0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1}, +}; + +#endif // SNAPPY_HAVE_SSSE3 + +// Copy [src, src+(op_limit-op)) to [op, (op_limit-op)) but faster than +// IncrementalCopySlow. buf_limit is the address past the end of the writable +// region of the buffer. +inline char* IncrementalCopy(const char* src, char* op, char* const op_limit, + char* const buf_limit) { + // Terminology: + // + // slop = buf_limit - op + // pat = op - src + // len = limit - op + assert(src < op); + assert(op <= op_limit); + assert(op_limit <= buf_limit); + // NOTE: The compressor always emits 4 <= len <= 64. It is ok to assume that + // to optimize this function but we have to also handle other cases in case + // the input does not satisfy these conditions. + + size_t pattern_size = op - src; + // The cases are split into different branches to allow the branch predictor, + // FDO, and static prediction hints to work better. For each input we list the + // ratio of invocations that match each condition. + // + // input slop < 16 pat < 8 len > 16 + // ------------------------------------------ + // html|html4|cp 0% 1.01% 27.73% + // urls 0% 0.88% 14.79% + // jpg 0% 64.29% 7.14% + // pdf 0% 2.56% 58.06% + // txt[1-4] 0% 0.23% 0.97% + // pb 0% 0.96% 13.88% + // bin 0.01% 22.27% 41.17% + // + // It is very rare that we don't have enough slop for doing block copies. It + // is also rare that we need to expand a pattern. Small patterns are common + // for incompressible formats and for those we are plenty fast already. + // Lengths are normally not greater than 16 but they vary depending on the + // input. In general if we always predict len <= 16 it would be an ok + // prediction. + // + // In order to be fast we want a pattern >= 8 bytes and an unrolled loop + // copying 2x 8 bytes at a time. + + // Handle the uncommon case where pattern is less than 8 bytes. + if (SNAPPY_PREDICT_FALSE(pattern_size < 8)) { +#if SNAPPY_HAVE_SSSE3 + // Load the first eight bytes into an 128-bit XMM register, then use PSHUFB + // to permute the register's contents in-place into a repeating sequence of + // the first "pattern_size" bytes. + // For example, suppose: + // src == "abc" + // op == op + 3 + // After _mm_shuffle_epi8(), "pattern" will have five copies of "abc" + // followed by one byte of slop: abcabcabcabcabca. + // + // The non-SSE fallback implementation suffers from store-forwarding stalls + // because its loads and stores partly overlap. By expanding the pattern + // in-place, we avoid the penalty. + if (SNAPPY_PREDICT_TRUE(op <= buf_limit - 16)) { + const __m128i shuffle_mask = _mm_load_si128( + reinterpret_cast(pshufb_fill_patterns) + + pattern_size - 1); + const __m128i pattern = _mm_shuffle_epi8( + _mm_loadl_epi64(reinterpret_cast(src)), shuffle_mask); + // Uninitialized bytes are masked out by the shuffle mask. + // TODO: remove annotation and macro defs once MSan is fixed. + SNAPPY_ANNOTATE_MEMORY_IS_INITIALIZED(&pattern, sizeof(pattern)); + pattern_size *= 16 / pattern_size; + char* op_end = std::min(op_limit, buf_limit - 15); + while (op < op_end) { + _mm_storeu_si128(reinterpret_cast<__m128i*>(op), pattern); + op += pattern_size; + } + if (SNAPPY_PREDICT_TRUE(op >= op_limit)) return op_limit; + } + return IncrementalCopySlow(src, op, op_limit); +#else // !SNAPPY_HAVE_SSSE3 + // If plenty of buffer space remains, expand the pattern to at least 8 + // bytes. The way the following loop is written, we need 8 bytes of buffer + // space if pattern_size >= 4, 11 bytes if pattern_size is 1 or 3, and 10 + // bytes if pattern_size is 2. Precisely encoding that is probably not + // worthwhile; instead, invoke the slow path if we cannot write 11 bytes + // (because 11 are required in the worst case). + if (SNAPPY_PREDICT_TRUE(op <= buf_limit - 11)) { + while (pattern_size < 8) { + UnalignedCopy64(src, op); + op += pattern_size; + pattern_size *= 2; + } + if (SNAPPY_PREDICT_TRUE(op >= op_limit)) return op_limit; + } else { + return IncrementalCopySlow(src, op, op_limit); + } +#endif // SNAPPY_HAVE_SSSE3 + } + assert(pattern_size >= 8); + + // Copy 2x 8 bytes at a time. Because op - src can be < 16, a single + // UnalignedCopy128 might overwrite data in op. UnalignedCopy64 is safe + // because expanding the pattern to at least 8 bytes guarantees that + // op - src >= 8. + // + // Typically, the op_limit is the gating factor so try to simplify the loop + // based on that. + if (SNAPPY_PREDICT_TRUE(op_limit <= buf_limit - 16)) { + // There is at least one, and at most four 16-byte blocks. Writing four + // conditionals instead of a loop allows FDO to layout the code with respect + // to the actual probabilities of each length. + // TODO: Replace with loop with trip count hint. + UnalignedCopy64(src, op); + UnalignedCopy64(src + 8, op + 8); + + if (op + 16 < op_limit) { + UnalignedCopy64(src + 16, op + 16); + UnalignedCopy64(src + 24, op + 24); + } + if (op + 32 < op_limit) { + UnalignedCopy64(src + 32, op + 32); + UnalignedCopy64(src + 40, op + 40); + } + if (op + 48 < op_limit) { + UnalignedCopy64(src + 48, op + 48); + UnalignedCopy64(src + 56, op + 56); + } + return op_limit; + } + + // Fall back to doing as much as we can with the available slop in the + // buffer. This code path is relatively cold however so we save code size by + // avoiding unrolling and vectorizing. + // + // TODO: Remove pragma when when cold regions don't get vectorized + // or unrolled. +#ifdef __clang__ +#pragma clang loop unroll(disable) +#endif + for (char *op_end = buf_limit - 16; op < op_end; op += 16, src += 16) { + UnalignedCopy64(src, op); + UnalignedCopy64(src + 8, op + 8); + } + if (op >= op_limit) + return op_limit; + + // We only take this branch if we didn't have enough slop and we can do a + // single 8 byte copy. + if (SNAPPY_PREDICT_FALSE(op <= buf_limit - 8)) { + UnalignedCopy64(src, op); + src += 8; + op += 8; + } + return IncrementalCopySlow(src, op, op_limit); +} + +} // namespace + +template +static inline char* EmitLiteral(char* op, + const char* literal, + int len) { + // The vast majority of copies are below 16 bytes, for which a + // call to memcpy is overkill. This fast path can sometimes + // copy up to 15 bytes too much, but that is okay in the + // main loop, since we have a bit to go on for both sides: + // + // - The input will always have kInputMarginBytes = 15 extra + // available bytes, as long as we're in the main loop, and + // if not, allow_fast_path = false. + // - The output will always have 32 spare bytes (see + // MaxCompressedLength). + assert(len > 0); // Zero-length literals are disallowed + int n = len - 1; + if (allow_fast_path && len <= 16) { + // Fits in tag byte + *op++ = LITERAL | (n << 2); + + UnalignedCopy128(literal, op); + return op + len; + } + + if (n < 60) { + // Fits in tag byte + *op++ = LITERAL | (n << 2); + } else { + int count = (Bits::Log2Floor(n) >> 3) + 1; + assert(count >= 1); + assert(count <= 4); + *op++ = LITERAL | ((59 + count) << 2); + // Encode in upcoming bytes. + // Write 4 bytes, though we may care about only 1 of them. The output buffer + // is guaranteed to have at least 3 more spaces left as 'len >= 61' holds + // here and there is a memcpy of size 'len' below. + LittleEndian::Store32(op, n); + op += count; + } + memcpy(op, literal, len); + return op + len; +} + +template +static inline char* EmitCopyAtMost64(char* op, size_t offset, size_t len) { + assert(len <= 64); + assert(len >= 4); + assert(offset < 65536); + assert(len_less_than_12 == (len < 12)); + + if (len_less_than_12 && SNAPPY_PREDICT_TRUE(offset < 2048)) { + // offset fits in 11 bits. The 3 highest go in the top of the first byte, + // and the rest go in the second byte. + *op++ = COPY_1_BYTE_OFFSET + ((len - 4) << 2) + ((offset >> 3) & 0xe0); + *op++ = offset & 0xff; + } else { + // Write 4 bytes, though we only care about 3 of them. The output buffer + // is required to have some slack, so the extra byte won't overrun it. + uint32 u = COPY_2_BYTE_OFFSET + ((len - 1) << 2) + (offset << 8); + LittleEndian::Store32(op, u); + op += 3; + } + return op; +} + +template +static inline char* EmitCopy(char* op, size_t offset, size_t len) { + assert(len_less_than_12 == (len < 12)); + if (len_less_than_12) { + return EmitCopyAtMost64(op, offset, len); + } else { + // A special case for len <= 64 might help, but so far measurements suggest + // it's in the noise. + + // Emit 64 byte copies but make sure to keep at least four bytes reserved. + while (SNAPPY_PREDICT_FALSE(len >= 68)) { + op = EmitCopyAtMost64(op, offset, 64); + len -= 64; + } + + // One or two copies will now finish the job. + if (len > 64) { + op = EmitCopyAtMost64(op, offset, 60); + len -= 60; + } + + // Emit remainder. + if (len < 12) { + op = EmitCopyAtMost64(op, offset, len); + } else { + op = EmitCopyAtMost64(op, offset, len); + } + return op; + } +} + +bool GetUncompressedLength(const char* start, size_t n, size_t* result) { + uint32 v = 0; + const char* limit = start + n; + if (Varint::Parse32WithLimit(start, limit, &v) != NULL) { + *result = v; + return true; + } else { + return false; + } +} + +namespace { +uint32 CalculateTableSize(uint32 input_size) { + static_assert( + kMaxHashTableSize >= kMinHashTableSize, + "kMaxHashTableSize should be greater or equal to kMinHashTableSize."); + if (input_size > kMaxHashTableSize) { + return kMaxHashTableSize; + } + if (input_size < kMinHashTableSize) { + return kMinHashTableSize; + } + // This is equivalent to Log2Ceiling(input_size), assuming input_size > 1. + // 2 << Log2Floor(x - 1) is equivalent to 1 << (1 + Log2Floor(x - 1)). + return 2u << Bits::Log2Floor(input_size - 1); +} +} // namespace + +namespace internal { +WorkingMemory::WorkingMemory(size_t input_size) { + const size_t max_fragment_size = std::min(input_size, kBlockSize); + const size_t table_size = CalculateTableSize(max_fragment_size); + size_ = table_size * sizeof(*table_) + max_fragment_size + + MaxCompressedLength(max_fragment_size); + mem_ = std::allocator().allocate(size_); + table_ = reinterpret_cast(mem_); + input_ = mem_ + table_size * sizeof(*table_); + output_ = input_ + max_fragment_size; +} + +WorkingMemory::~WorkingMemory() { + std::allocator().deallocate(mem_, size_); +} + +uint16* WorkingMemory::GetHashTable(size_t fragment_size, + int* table_size) const { + const size_t htsize = CalculateTableSize(fragment_size); + memset(table_, 0, htsize * sizeof(*table_)); + *table_size = htsize; + return table_; +} +} // end namespace internal + +// For 0 <= offset <= 4, GetUint32AtOffset(GetEightBytesAt(p), offset) will +// equal UNALIGNED_LOAD32(p + offset). Motivation: On x86-64 hardware we have +// empirically found that overlapping loads such as +// UNALIGNED_LOAD32(p) ... UNALIGNED_LOAD32(p+1) ... UNALIGNED_LOAD32(p+2) +// are slower than UNALIGNED_LOAD64(p) followed by shifts and casts to uint32. +// +// We have different versions for 64- and 32-bit; ideally we would avoid the +// two functions and just inline the UNALIGNED_LOAD64 call into +// GetUint32AtOffset, but GCC (at least not as of 4.6) is seemingly not clever +// enough to avoid loading the value multiple times then. For 64-bit, the load +// is done when GetEightBytesAt() is called, whereas for 32-bit, the load is +// done at GetUint32AtOffset() time. + +#ifdef ARCH_K8 + +typedef uint64 EightBytesReference; + +static inline EightBytesReference GetEightBytesAt(const char* ptr) { + return UNALIGNED_LOAD64(ptr); +} + +static inline uint32 GetUint32AtOffset(uint64 v, int offset) { + assert(offset >= 0); + assert(offset <= 4); + return v >> (LittleEndian::IsLittleEndian() ? 8 * offset : 32 - 8 * offset); +} + +#else + +typedef const char* EightBytesReference; + +static inline EightBytesReference GetEightBytesAt(const char* ptr) { + return ptr; +} + +static inline uint32 GetUint32AtOffset(const char* v, int offset) { + assert(offset >= 0); + assert(offset <= 4); + return UNALIGNED_LOAD32(v + offset); +} + +#endif + +// Flat array compression that does not emit the "uncompressed length" +// prefix. Compresses "input" string to the "*op" buffer. +// +// REQUIRES: "input" is at most "kBlockSize" bytes long. +// REQUIRES: "op" points to an array of memory that is at least +// "MaxCompressedLength(input.size())" in size. +// REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. +// REQUIRES: "table_size" is a power of two +// +// Returns an "end" pointer into "op" buffer. +// "end - op" is the compressed size of "input". +namespace internal { +char* CompressFragment(const char* input, + size_t input_size, + char* op, + uint16* table, + const int table_size) { + // "ip" is the input pointer, and "op" is the output pointer. + const char* ip = input; + assert(input_size <= kBlockSize); + assert((table_size & (table_size - 1)) == 0); // table must be power of two + const int shift = 32 - Bits::Log2Floor(table_size); + assert(static_cast(kuint32max >> shift) == table_size - 1); + const char* ip_end = input + input_size; + const char* base_ip = ip; + // Bytes in [next_emit, ip) will be emitted as literal bytes. Or + // [next_emit, ip_end) after the main loop. + const char* next_emit = ip; + + const size_t kInputMarginBytes = 15; + if (SNAPPY_PREDICT_TRUE(input_size >= kInputMarginBytes)) { + const char* ip_limit = input + input_size - kInputMarginBytes; + + for (uint32 next_hash = Hash(++ip, shift); ; ) { + assert(next_emit < ip); + // The body of this loop calls EmitLiteral once and then EmitCopy one or + // more times. (The exception is that when we're close to exhausting + // the input we goto emit_remainder.) + // + // In the first iteration of this loop we're just starting, so + // there's nothing to copy, so calling EmitLiteral once is + // necessary. And we only start a new iteration when the + // current iteration has determined that a call to EmitLiteral will + // precede the next call to EmitCopy (if any). + // + // Step 1: Scan forward in the input looking for a 4-byte-long match. + // If we get close to exhausting the input then goto emit_remainder. + // + // Heuristic match skipping: If 32 bytes are scanned with no matches + // found, start looking only at every other byte. If 32 more bytes are + // scanned (or skipped), look at every third byte, etc.. When a match is + // found, immediately go back to looking at every byte. This is a small + // loss (~5% performance, ~0.1% density) for compressible data due to more + // bookkeeping, but for non-compressible data (such as JPEG) it's a huge + // win since the compressor quickly "realizes" the data is incompressible + // and doesn't bother looking for matches everywhere. + // + // The "skip" variable keeps track of how many bytes there are since the + // last match; dividing it by 32 (ie. right-shifting by five) gives the + // number of bytes to move ahead for each iteration. + uint32 skip = 32; + + const char* next_ip = ip; + const char* candidate; + do { + ip = next_ip; + uint32 hash = next_hash; + assert(hash == Hash(ip, shift)); + uint32 bytes_between_hash_lookups = skip >> 5; + skip += bytes_between_hash_lookups; + next_ip = ip + bytes_between_hash_lookups; + if (SNAPPY_PREDICT_FALSE(next_ip > ip_limit)) { + goto emit_remainder; + } + next_hash = Hash(next_ip, shift); + candidate = base_ip + table[hash]; + assert(candidate >= base_ip); + assert(candidate < ip); + + table[hash] = ip - base_ip; + } while (SNAPPY_PREDICT_TRUE(UNALIGNED_LOAD32(ip) != + UNALIGNED_LOAD32(candidate))); + + // Step 2: A 4-byte match has been found. We'll later see if more + // than 4 bytes match. But, prior to the match, input + // bytes [next_emit, ip) are unmatched. Emit them as "literal bytes." + assert(next_emit + 16 <= ip_end); + op = EmitLiteral(op, next_emit, ip - next_emit); + + // Step 3: Call EmitCopy, and then see if another EmitCopy could + // be our next move. Repeat until we find no match for the + // input immediately after what was consumed by the last EmitCopy call. + // + // If we exit this loop normally then we need to call EmitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can exit + // this loop via goto if we get close to exhausting the input. + EightBytesReference input_bytes; + uint32 candidate_bytes = 0; + + do { + // We have a 4-byte match at ip, and no need to emit any + // "literal bytes" prior to ip. + const char* base = ip; + std::pair p = + FindMatchLength(candidate + 4, ip + 4, ip_end); + size_t matched = 4 + p.first; + ip += matched; + size_t offset = base - candidate; + assert(0 == memcmp(base, candidate, matched)); + if (p.second) { + op = EmitCopy(op, offset, matched); + } else { + op = EmitCopy(op, offset, matched); + } + next_emit = ip; + if (SNAPPY_PREDICT_FALSE(ip >= ip_limit)) { + goto emit_remainder; + } + // We are now looking for a 4-byte match again. We read + // table[Hash(ip, shift)] for that. To improve compression, + // we also update table[Hash(ip - 1, shift)] and table[Hash(ip, shift)]. + input_bytes = GetEightBytesAt(ip - 1); + uint32 prev_hash = HashBytes(GetUint32AtOffset(input_bytes, 0), shift); + table[prev_hash] = ip - base_ip - 1; + uint32 cur_hash = HashBytes(GetUint32AtOffset(input_bytes, 1), shift); + candidate = base_ip + table[cur_hash]; + candidate_bytes = UNALIGNED_LOAD32(candidate); + table[cur_hash] = ip - base_ip; + } while (GetUint32AtOffset(input_bytes, 1) == candidate_bytes); + + next_hash = HashBytes(GetUint32AtOffset(input_bytes, 2), shift); + ++ip; + } + } + + emit_remainder: + // Emit the remaining bytes as a literal + if (next_emit < ip_end) { + op = EmitLiteral(op, next_emit, + ip_end - next_emit); + } + + return op; +} +} // end namespace internal + +// Called back at avery compression call to trace parameters and sizes. +static inline void Report(const char *algorithm, size_t compressed_size, + size_t uncompressed_size) {} + +// Signature of output types needed by decompression code. +// The decompression code is templatized on a type that obeys this +// signature so that we do not pay virtual function call overhead in +// the middle of a tight decompression loop. +// +// class DecompressionWriter { +// public: +// // Called before decompression +// void SetExpectedLength(size_t length); +// +// // Called after decompression +// bool CheckLength() const; +// +// // Called repeatedly during decompression +// bool Append(const char* ip, size_t length); +// bool AppendFromSelf(uint32 offset, size_t length); +// +// // The rules for how TryFastAppend differs from Append are somewhat +// // convoluted: +// // +// // - TryFastAppend is allowed to decline (return false) at any +// // time, for any reason -- just "return false" would be +// // a perfectly legal implementation of TryFastAppend. +// // The intention is for TryFastAppend to allow a fast path +// // in the common case of a small append. +// // - TryFastAppend is allowed to read up to bytes +// // from the input buffer, whereas Append is allowed to read +// // . However, if it returns true, it must leave +// // at least five (kMaximumTagLength) bytes in the input buffer +// // afterwards, so that there is always enough space to read the +// // next tag without checking for a refill. +// // - TryFastAppend must always return decline (return false) +// // if is 61 or more, as in this case the literal length is not +// // decoded fully. In practice, this should not be a big problem, +// // as it is unlikely that one would implement a fast path accepting +// // this much data. +// // +// bool TryFastAppend(const char* ip, size_t available, size_t length); +// }; + +static inline uint32 ExtractLowBytes(uint32 v, int n) { + assert(n >= 0); + assert(n <= 4); +#if SNAPPY_HAVE_BMI2 + return _bzhi_u32(v, 8 * n); +#else + // This needs to be wider than uint32 otherwise `mask << 32` will be + // undefined. + uint64 mask = 0xffffffff; + return v & ~(mask << (8 * n)); +#endif +} + +static inline bool LeftShiftOverflows(uint8 value, uint32 shift) { + assert(shift < 32); + static const uint8 masks[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe}; + return (value & masks[shift]) != 0; +} + +// Helper class for decompression +class SnappyDecompressor { + private: + Source* reader_; // Underlying source of bytes to decompress + const char* ip_; // Points to next buffered byte + const char* ip_limit_; // Points just past buffered bytes + uint32 peeked_; // Bytes peeked from reader (need to skip) + bool eof_; // Hit end of input without an error? + char scratch_[kMaximumTagLength]; // See RefillTag(). + + // Ensure that all of the tag metadata for the next tag is available + // in [ip_..ip_limit_-1]. Also ensures that [ip,ip+4] is readable even + // if (ip_limit_ - ip_ < 5). + // + // Returns true on success, false on error or end of input. + bool RefillTag(); + + public: + explicit SnappyDecompressor(Source* reader) + : reader_(reader), + ip_(NULL), + ip_limit_(NULL), + peeked_(0), + eof_(false) { + } + + ~SnappyDecompressor() { + // Advance past any bytes we peeked at from the reader + reader_->Skip(peeked_); + } + + // Returns true iff we have hit the end of the input without an error. + bool eof() const { + return eof_; + } + + // Read the uncompressed length stored at the start of the compressed data. + // On success, stores the length in *result and returns true. + // On failure, returns false. + bool ReadUncompressedLength(uint32* result) { + assert(ip_ == NULL); // Must not have read anything yet + // Length is encoded in 1..5 bytes + *result = 0; + uint32 shift = 0; + while (true) { + if (shift >= 32) return false; + size_t n; + const char* ip = reader_->Peek(&n); + if (n == 0) return false; + const unsigned char c = *(reinterpret_cast(ip)); + reader_->Skip(1); + uint32 val = c & 0x7f; + if (LeftShiftOverflows(static_cast(val), shift)) return false; + *result |= val << shift; + if (c < 128) { + break; + } + shift += 7; + } + return true; + } + + // Process the next item found in the input. + // Returns true if successful, false on error or end of input. + template +#if defined(__GNUC__) && defined(__x86_64__) + __attribute__((aligned(32))) +#endif + void DecompressAllTags(Writer* writer) { + // In x86, pad the function body to start 16 bytes later. This function has + // a couple of hotspots that are highly sensitive to alignment: we have + // observed regressions by more than 20% in some metrics just by moving the + // exact same code to a different position in the benchmark binary. + // + // Putting this code on a 32-byte-aligned boundary + 16 bytes makes us hit + // the "lucky" case consistently. Unfortunately, this is a very brittle + // workaround, and future differences in code generation may reintroduce + // this regression. If you experience a big, difficult to explain, benchmark + // performance regression here, first try removing this hack. +#if defined(__GNUC__) && defined(__x86_64__) + // Two 8-byte "NOP DWORD ptr [EAX + EAX*1 + 00000000H]" instructions. + asm(".byte 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00"); + asm(".byte 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00"); +#endif + + const char* ip = ip_; + // We could have put this refill fragment only at the beginning of the loop. + // However, duplicating it at the end of each branch gives the compiler more + // scope to optimize the expression based on the local + // context, which overall increases speed. + #define MAYBE_REFILL() \ + if (ip_limit_ - ip < kMaximumTagLength) { \ + ip_ = ip; \ + if (!RefillTag()) return; \ + ip = ip_; \ + } + + MAYBE_REFILL(); + for ( ;; ) { + const unsigned char c = *(reinterpret_cast(ip++)); + + // Ratio of iterations that have LITERAL vs non-LITERAL for different + // inputs. + // + // input LITERAL NON_LITERAL + // ----------------------------------- + // html|html4|cp 23% 77% + // urls 36% 64% + // jpg 47% 53% + // pdf 19% 81% + // txt[1-4] 25% 75% + // pb 24% 76% + // bin 24% 76% + if (SNAPPY_PREDICT_FALSE((c & 0x3) == LITERAL)) { + size_t literal_length = (c >> 2) + 1u; + if (writer->TryFastAppend(ip, ip_limit_ - ip, literal_length)) { + assert(literal_length < 61); + ip += literal_length; + // NOTE: There is no MAYBE_REFILL() here, as TryFastAppend() + // will not return true unless there's already at least five spare + // bytes in addition to the literal. + continue; + } + if (SNAPPY_PREDICT_FALSE(literal_length >= 61)) { + // Long literal. + const size_t literal_length_length = literal_length - 60; + literal_length = + ExtractLowBytes(LittleEndian::Load32(ip), literal_length_length) + + 1; + ip += literal_length_length; + } + + size_t avail = ip_limit_ - ip; + while (avail < literal_length) { + if (!writer->Append(ip, avail)) return; + literal_length -= avail; + reader_->Skip(peeked_); + size_t n; + ip = reader_->Peek(&n); + avail = n; + peeked_ = avail; + if (avail == 0) return; // Premature end of input + ip_limit_ = ip + avail; + } + if (!writer->Append(ip, literal_length)) { + return; + } + ip += literal_length; + MAYBE_REFILL(); + } else { + const size_t entry = char_table[c]; + const size_t trailer = + ExtractLowBytes(LittleEndian::Load32(ip), entry >> 11); + const size_t length = entry & 0xff; + ip += entry >> 11; + + // copy_offset/256 is encoded in bits 8..10. By just fetching + // those bits, we get copy_offset (since the bit-field starts at + // bit 8). + const size_t copy_offset = entry & 0x700; + if (!writer->AppendFromSelf(copy_offset + trailer, length)) { + return; + } + MAYBE_REFILL(); + } + } + +#undef MAYBE_REFILL + } +}; + +bool SnappyDecompressor::RefillTag() { + const char* ip = ip_; + if (ip == ip_limit_) { + // Fetch a new fragment from the reader + reader_->Skip(peeked_); // All peeked bytes are used up + size_t n; + ip = reader_->Peek(&n); + peeked_ = n; + eof_ = (n == 0); + if (eof_) return false; + ip_limit_ = ip + n; + } + + // Read the tag character + assert(ip < ip_limit_); + const unsigned char c = *(reinterpret_cast(ip)); + const uint32 entry = char_table[c]; + const uint32 needed = (entry >> 11) + 1; // +1 byte for 'c' + assert(needed <= sizeof(scratch_)); + + // Read more bytes from reader if needed + uint32 nbuf = ip_limit_ - ip; + if (nbuf < needed) { + // Stitch together bytes from ip and reader to form the word + // contents. We store the needed bytes in "scratch_". They + // will be consumed immediately by the caller since we do not + // read more than we need. + memmove(scratch_, ip, nbuf); + reader_->Skip(peeked_); // All peeked bytes are used up + peeked_ = 0; + while (nbuf < needed) { + size_t length; + const char* src = reader_->Peek(&length); + if (length == 0) return false; + uint32 to_add = std::min(needed - nbuf, length); + memcpy(scratch_ + nbuf, src, to_add); + nbuf += to_add; + reader_->Skip(to_add); + } + assert(nbuf == needed); + ip_ = scratch_; + ip_limit_ = scratch_ + needed; + } else if (nbuf < kMaximumTagLength) { + // Have enough bytes, but move into scratch_ so that we do not + // read past end of input + memmove(scratch_, ip, nbuf); + reader_->Skip(peeked_); // All peeked bytes are used up + peeked_ = 0; + ip_ = scratch_; + ip_limit_ = scratch_ + nbuf; + } else { + // Pass pointer to buffer returned by reader_. + ip_ = ip; + } + return true; +} + +template +static bool InternalUncompress(Source* r, Writer* writer) { + // Read the uncompressed length from the front of the compressed input + SnappyDecompressor decompressor(r); + uint32 uncompressed_len = 0; + if (!decompressor.ReadUncompressedLength(&uncompressed_len)) return false; + + return InternalUncompressAllTags(&decompressor, writer, r->Available(), + uncompressed_len); +} + +template +static bool InternalUncompressAllTags(SnappyDecompressor* decompressor, + Writer* writer, + uint32 compressed_len, + uint32 uncompressed_len) { + Report("snappy_uncompress", compressed_len, uncompressed_len); + + writer->SetExpectedLength(uncompressed_len); + + // Process the entire input + decompressor->DecompressAllTags(writer); + writer->Flush(); + return (decompressor->eof() && writer->CheckLength()); +} + +bool GetUncompressedLength(Source* source, uint32* result) { + SnappyDecompressor decompressor(source); + return decompressor.ReadUncompressedLength(result); +} + +size_t Compress(Source* reader, Sink* writer) { + size_t written = 0; + size_t N = reader->Available(); + const size_t uncompressed_size = N; + char ulength[Varint::kMax32]; + char* p = Varint::Encode32(ulength, N); + writer->Append(ulength, p-ulength); + written += (p - ulength); + + internal::WorkingMemory wmem(N); + + while (N > 0) { + // Get next block to compress (without copying if possible) + size_t fragment_size; + const char* fragment = reader->Peek(&fragment_size); + assert(fragment_size != 0); // premature end of input + const size_t num_to_read = std::min(N, kBlockSize); + size_t bytes_read = fragment_size; + + size_t pending_advance = 0; + if (bytes_read >= num_to_read) { + // Buffer returned by reader is large enough + pending_advance = num_to_read; + fragment_size = num_to_read; + } else { + char* scratch = wmem.GetScratchInput(); + memcpy(scratch, fragment, bytes_read); + reader->Skip(bytes_read); + + while (bytes_read < num_to_read) { + fragment = reader->Peek(&fragment_size); + size_t n = std::min(fragment_size, num_to_read - bytes_read); + memcpy(scratch + bytes_read, fragment, n); + bytes_read += n; + reader->Skip(n); + } + assert(bytes_read == num_to_read); + fragment = scratch; + fragment_size = num_to_read; + } + assert(fragment_size == num_to_read); + + // Get encoding table for compression + int table_size; + uint16* table = wmem.GetHashTable(num_to_read, &table_size); + + // Compress input_fragment and append to dest + const int max_output = MaxCompressedLength(num_to_read); + + // Need a scratch buffer for the output, in case the byte sink doesn't + // have room for us directly. + + // Since we encode kBlockSize regions followed by a region + // which is <= kBlockSize in length, a previously allocated + // scratch_output[] region is big enough for this iteration. + char* dest = writer->GetAppendBuffer(max_output, wmem.GetScratchOutput()); + char* end = internal::CompressFragment(fragment, fragment_size, dest, table, + table_size); + writer->Append(dest, end - dest); + written += (end - dest); + + N -= num_to_read; + reader->Skip(pending_advance); + } + + Report("snappy_compress", written, uncompressed_size); + + return written; +} + +// ----------------------------------------------------------------------- +// IOVec interfaces +// ----------------------------------------------------------------------- + +// A type that writes to an iovec. +// Note that this is not a "ByteSink", but a type that matches the +// Writer template argument to SnappyDecompressor::DecompressAllTags(). +class SnappyIOVecWriter { + private: + // output_iov_end_ is set to iov + count and used to determine when + // the end of the iovs is reached. + const struct iovec* output_iov_end_; + +#if !defined(NDEBUG) + const struct iovec* output_iov_; +#endif // !defined(NDEBUG) + + // Current iov that is being written into. + const struct iovec* curr_iov_; + + // Pointer to current iov's write location. + char* curr_iov_output_; + + // Remaining bytes to write into curr_iov_output. + size_t curr_iov_remaining_; + + // Total bytes decompressed into output_iov_ so far. + size_t total_written_; + + // Maximum number of bytes that will be decompressed into output_iov_. + size_t output_limit_; + + static inline char* GetIOVecPointer(const struct iovec* iov, size_t offset) { + return reinterpret_cast(iov->iov_base) + offset; + } + + public: + // Does not take ownership of iov. iov must be valid during the + // entire lifetime of the SnappyIOVecWriter. + inline SnappyIOVecWriter(const struct iovec* iov, size_t iov_count) + : output_iov_end_(iov + iov_count), +#if !defined(NDEBUG) + output_iov_(iov), +#endif // !defined(NDEBUG) + curr_iov_(iov), + curr_iov_output_(iov_count ? reinterpret_cast(iov->iov_base) + : nullptr), + curr_iov_remaining_(iov_count ? iov->iov_len : 0), + total_written_(0), + output_limit_(-1) {} + + inline void SetExpectedLength(size_t len) { + output_limit_ = len; + } + + inline bool CheckLength() const { + return total_written_ == output_limit_; + } + + inline bool Append(const char* ip, size_t len) { + if (total_written_ + len > output_limit_) { + return false; + } + + return AppendNoCheck(ip, len); + } + + inline bool AppendNoCheck(const char* ip, size_t len) { + while (len > 0) { + if (curr_iov_remaining_ == 0) { + // This iovec is full. Go to the next one. + if (curr_iov_ + 1 >= output_iov_end_) { + return false; + } + ++curr_iov_; + curr_iov_output_ = reinterpret_cast(curr_iov_->iov_base); + curr_iov_remaining_ = curr_iov_->iov_len; + } + + const size_t to_write = std::min(len, curr_iov_remaining_); + memcpy(curr_iov_output_, ip, to_write); + curr_iov_output_ += to_write; + curr_iov_remaining_ -= to_write; + total_written_ += to_write; + ip += to_write; + len -= to_write; + } + + return true; + } + + inline bool TryFastAppend(const char* ip, size_t available, size_t len) { + const size_t space_left = output_limit_ - total_written_; + if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16 && + curr_iov_remaining_ >= 16) { + // Fast path, used for the majority (about 95%) of invocations. + UnalignedCopy128(ip, curr_iov_output_); + curr_iov_output_ += len; + curr_iov_remaining_ -= len; + total_written_ += len; + return true; + } + + return false; + } + + inline bool AppendFromSelf(size_t offset, size_t len) { + // See SnappyArrayWriter::AppendFromSelf for an explanation of + // the "offset - 1u" trick. + if (offset - 1u >= total_written_) { + return false; + } + const size_t space_left = output_limit_ - total_written_; + if (len > space_left) { + return false; + } + + // Locate the iovec from which we need to start the copy. + const iovec* from_iov = curr_iov_; + size_t from_iov_offset = curr_iov_->iov_len - curr_iov_remaining_; + while (offset > 0) { + if (from_iov_offset >= offset) { + from_iov_offset -= offset; + break; + } + + offset -= from_iov_offset; + --from_iov; +#if !defined(NDEBUG) + assert(from_iov >= output_iov_); +#endif // !defined(NDEBUG) + from_iov_offset = from_iov->iov_len; + } + + // Copy bytes starting from the iovec pointed to by from_iov_index to + // the current iovec. + while (len > 0) { + assert(from_iov <= curr_iov_); + if (from_iov != curr_iov_) { + const size_t to_copy = + std::min(from_iov->iov_len - from_iov_offset, len); + AppendNoCheck(GetIOVecPointer(from_iov, from_iov_offset), to_copy); + len -= to_copy; + if (len > 0) { + ++from_iov; + from_iov_offset = 0; + } + } else { + size_t to_copy = curr_iov_remaining_; + if (to_copy == 0) { + // This iovec is full. Go to the next one. + if (curr_iov_ + 1 >= output_iov_end_) { + return false; + } + ++curr_iov_; + curr_iov_output_ = reinterpret_cast(curr_iov_->iov_base); + curr_iov_remaining_ = curr_iov_->iov_len; + continue; + } + if (to_copy > len) { + to_copy = len; + } + + IncrementalCopy(GetIOVecPointer(from_iov, from_iov_offset), + curr_iov_output_, curr_iov_output_ + to_copy, + curr_iov_output_ + curr_iov_remaining_); + curr_iov_output_ += to_copy; + curr_iov_remaining_ -= to_copy; + from_iov_offset += to_copy; + total_written_ += to_copy; + len -= to_copy; + } + } + + return true; + } + + inline void Flush() {} +}; + +bool RawUncompressToIOVec(const char* compressed, size_t compressed_length, + const struct iovec* iov, size_t iov_cnt) { + ByteArraySource reader(compressed, compressed_length); + return RawUncompressToIOVec(&reader, iov, iov_cnt); +} + +bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov, + size_t iov_cnt) { + SnappyIOVecWriter output(iov, iov_cnt); + return InternalUncompress(compressed, &output); +} + +// ----------------------------------------------------------------------- +// Flat array interfaces +// ----------------------------------------------------------------------- + +// A type that writes to a flat array. +// Note that this is not a "ByteSink", but a type that matches the +// Writer template argument to SnappyDecompressor::DecompressAllTags(). +class SnappyArrayWriter { + private: + char* base_; + char* op_; + char* op_limit_; + + public: + inline explicit SnappyArrayWriter(char* dst) + : base_(dst), + op_(dst), + op_limit_(dst) { + } + + inline void SetExpectedLength(size_t len) { + op_limit_ = op_ + len; + } + + inline bool CheckLength() const { + return op_ == op_limit_; + } + + inline bool Append(const char* ip, size_t len) { + char* op = op_; + const size_t space_left = op_limit_ - op; + if (space_left < len) { + return false; + } + memcpy(op, ip, len); + op_ = op + len; + return true; + } + + inline bool TryFastAppend(const char* ip, size_t available, size_t len) { + char* op = op_; + const size_t space_left = op_limit_ - op; + if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16) { + // Fast path, used for the majority (about 95%) of invocations. + UnalignedCopy128(ip, op); + op_ = op + len; + return true; + } else { + return false; + } + } + + inline bool AppendFromSelf(size_t offset, size_t len) { + char* const op_end = op_ + len; + + // Check if we try to append from before the start of the buffer. + // Normally this would just be a check for "produced < offset", + // but "produced <= offset - 1u" is equivalent for every case + // except the one where offset==0, where the right side will wrap around + // to a very big number. This is convenient, as offset==0 is another + // invalid case that we also want to catch, so that we do not go + // into an infinite loop. + if (Produced() <= offset - 1u || op_end > op_limit_) return false; + op_ = IncrementalCopy(op_ - offset, op_, op_end, op_limit_); + + return true; + } + inline size_t Produced() const { + assert(op_ >= base_); + return op_ - base_; + } + inline void Flush() {} +}; + +bool RawUncompress(const char* compressed, size_t n, char* uncompressed) { + ByteArraySource reader(compressed, n); + return RawUncompress(&reader, uncompressed); +} + +bool RawUncompress(Source* compressed, char* uncompressed) { + SnappyArrayWriter output(uncompressed); + return InternalUncompress(compressed, &output); +} + +bool Uncompress(const char* compressed, size_t n, std::string* uncompressed) { + size_t ulength; + if (!GetUncompressedLength(compressed, n, &ulength)) { + return false; + } + // On 32-bit builds: max_size() < kuint32max. Check for that instead + // of crashing (e.g., consider externally specified compressed data). + if (ulength > uncompressed->max_size()) { + return false; + } + STLStringResizeUninitialized(uncompressed, ulength); + return RawUncompress(compressed, n, string_as_array(uncompressed)); +} + +// A Writer that drops everything on the floor and just does validation +class SnappyDecompressionValidator { + private: + size_t expected_; + size_t produced_; + + public: + inline SnappyDecompressionValidator() : expected_(0), produced_(0) { } + inline void SetExpectedLength(size_t len) { + expected_ = len; + } + inline bool CheckLength() const { + return expected_ == produced_; + } + inline bool Append(const char* ip, size_t len) { + produced_ += len; + return produced_ <= expected_; + } + inline bool TryFastAppend(const char* ip, size_t available, size_t length) { + return false; + } + inline bool AppendFromSelf(size_t offset, size_t len) { + // See SnappyArrayWriter::AppendFromSelf for an explanation of + // the "offset - 1u" trick. + if (produced_ <= offset - 1u) return false; + produced_ += len; + return produced_ <= expected_; + } + inline void Flush() {} +}; + +bool IsValidCompressedBuffer(const char* compressed, size_t n) { + ByteArraySource reader(compressed, n); + SnappyDecompressionValidator writer; + return InternalUncompress(&reader, &writer); +} + +bool IsValidCompressed(Source* compressed) { + SnappyDecompressionValidator writer; + return InternalUncompress(compressed, &writer); +} + +void RawCompress(const char* input, + size_t input_length, + char* compressed, + size_t* compressed_length) { + ByteArraySource reader(input, input_length); + UncheckedByteArraySink writer(compressed); + Compress(&reader, &writer); + + // Compute how many bytes were added + *compressed_length = (writer.CurrentDestination() - compressed); +} + +size_t Compress(const char* input, size_t input_length, + std::string* compressed) { + // Pre-grow the buffer to the max length of the compressed output + STLStringResizeUninitialized(compressed, MaxCompressedLength(input_length)); + + size_t compressed_length; + RawCompress(input, input_length, string_as_array(compressed), + &compressed_length); + compressed->resize(compressed_length); + return compressed_length; +} + +// ----------------------------------------------------------------------- +// Sink interface +// ----------------------------------------------------------------------- + +// A type that decompresses into a Sink. The template parameter +// Allocator must export one method "char* Allocate(int size);", which +// allocates a buffer of "size" and appends that to the destination. +template +class SnappyScatteredWriter { + Allocator allocator_; + + // We need random access into the data generated so far. Therefore + // we keep track of all of the generated data as an array of blocks. + // All of the blocks except the last have length kBlockSize. + std::vector blocks_; + size_t expected_; + + // Total size of all fully generated blocks so far + size_t full_size_; + + // Pointer into current output block + char* op_base_; // Base of output block + char* op_ptr_; // Pointer to next unfilled byte in block + char* op_limit_; // Pointer just past block + + inline size_t Size() const { + return full_size_ + (op_ptr_ - op_base_); + } + + bool SlowAppend(const char* ip, size_t len); + bool SlowAppendFromSelf(size_t offset, size_t len); + + public: + inline explicit SnappyScatteredWriter(const Allocator& allocator) + : allocator_(allocator), + full_size_(0), + op_base_(NULL), + op_ptr_(NULL), + op_limit_(NULL) { + } + + inline void SetExpectedLength(size_t len) { + assert(blocks_.empty()); + expected_ = len; + } + + inline bool CheckLength() const { + return Size() == expected_; + } + + // Return the number of bytes actually uncompressed so far + inline size_t Produced() const { + return Size(); + } + + inline bool Append(const char* ip, size_t len) { + size_t avail = op_limit_ - op_ptr_; + if (len <= avail) { + // Fast path + memcpy(op_ptr_, ip, len); + op_ptr_ += len; + return true; + } else { + return SlowAppend(ip, len); + } + } + + inline bool TryFastAppend(const char* ip, size_t available, size_t length) { + char* op = op_ptr_; + const int space_left = op_limit_ - op; + if (length <= 16 && available >= 16 + kMaximumTagLength && + space_left >= 16) { + // Fast path, used for the majority (about 95%) of invocations. + UnalignedCopy128(ip, op); + op_ptr_ = op + length; + return true; + } else { + return false; + } + } + + inline bool AppendFromSelf(size_t offset, size_t len) { + char* const op_end = op_ptr_ + len; + // See SnappyArrayWriter::AppendFromSelf for an explanation of + // the "offset - 1u" trick. + if (SNAPPY_PREDICT_TRUE(offset - 1u < op_ptr_ - op_base_ && + op_end <= op_limit_)) { + // Fast path: src and dst in current block. + op_ptr_ = IncrementalCopy(op_ptr_ - offset, op_ptr_, op_end, op_limit_); + return true; + } + return SlowAppendFromSelf(offset, len); + } + + // Called at the end of the decompress. We ask the allocator + // write all blocks to the sink. + inline void Flush() { allocator_.Flush(Produced()); } +}; + +template +bool SnappyScatteredWriter::SlowAppend(const char* ip, size_t len) { + size_t avail = op_limit_ - op_ptr_; + while (len > avail) { + // Completely fill this block + memcpy(op_ptr_, ip, avail); + op_ptr_ += avail; + assert(op_limit_ - op_ptr_ == 0); + full_size_ += (op_ptr_ - op_base_); + len -= avail; + ip += avail; + + // Bounds check + if (full_size_ + len > expected_) { + return false; + } + + // Make new block + size_t bsize = std::min(kBlockSize, expected_ - full_size_); + op_base_ = allocator_.Allocate(bsize); + op_ptr_ = op_base_; + op_limit_ = op_base_ + bsize; + blocks_.push_back(op_base_); + avail = bsize; + } + + memcpy(op_ptr_, ip, len); + op_ptr_ += len; + return true; +} + +template +bool SnappyScatteredWriter::SlowAppendFromSelf(size_t offset, + size_t len) { + // Overflow check + // See SnappyArrayWriter::AppendFromSelf for an explanation of + // the "offset - 1u" trick. + const size_t cur = Size(); + if (offset - 1u >= cur) return false; + if (expected_ - cur < len) return false; + + // Currently we shouldn't ever hit this path because Compress() chops the + // input into blocks and does not create cross-block copies. However, it is + // nice if we do not rely on that, since we can get better compression if we + // allow cross-block copies and thus might want to change the compressor in + // the future. + size_t src = cur - offset; + while (len-- > 0) { + char c = blocks_[src >> kBlockLog][src & (kBlockSize-1)]; + Append(&c, 1); + src++; + } + return true; +} + +class SnappySinkAllocator { + public: + explicit SnappySinkAllocator(Sink* dest): dest_(dest) {} + ~SnappySinkAllocator() {} + + char* Allocate(int size) { + Datablock block(new char[size], size); + blocks_.push_back(block); + return block.data; + } + + // We flush only at the end, because the writer wants + // random access to the blocks and once we hand the + // block over to the sink, we can't access it anymore. + // Also we don't write more than has been actually written + // to the blocks. + void Flush(size_t size) { + size_t size_written = 0; + size_t block_size; + for (int i = 0; i < blocks_.size(); ++i) { + block_size = std::min(blocks_[i].size, size - size_written); + dest_->AppendAndTakeOwnership(blocks_[i].data, block_size, + &SnappySinkAllocator::Deleter, NULL); + size_written += block_size; + } + blocks_.clear(); + } + + private: + struct Datablock { + char* data; + size_t size; + Datablock(char* p, size_t s) : data(p), size(s) {} + }; + + static void Deleter(void* arg, const char* bytes, size_t size) { + delete[] bytes; + } + + Sink* dest_; + std::vector blocks_; + + // Note: copying this object is allowed +}; + +size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed) { + SnappySinkAllocator allocator(uncompressed); + SnappyScatteredWriter writer(allocator); + InternalUncompress(compressed, &writer); + return writer.Produced(); +} + +bool Uncompress(Source* compressed, Sink* uncompressed) { + // Read the uncompressed length from the front of the compressed input + SnappyDecompressor decompressor(compressed); + uint32 uncompressed_len = 0; + if (!decompressor.ReadUncompressedLength(&uncompressed_len)) { + return false; + } + + char c; + size_t allocated_size; + char* buf = uncompressed->GetAppendBufferVariable( + 1, uncompressed_len, &c, 1, &allocated_size); + + const size_t compressed_len = compressed->Available(); + // If we can get a flat buffer, then use it, otherwise do block by block + // uncompression + if (allocated_size >= uncompressed_len) { + SnappyArrayWriter writer(buf); + bool result = InternalUncompressAllTags(&decompressor, &writer, + compressed_len, uncompressed_len); + uncompressed->Append(buf, writer.Produced()); + return result; + } else { + SnappySinkAllocator allocator(uncompressed); + SnappyScatteredWriter writer(allocator); + return InternalUncompressAllTags(&decompressor, &writer, compressed_len, + uncompressed_len); + } +} + +} // namespace snappy diff --git a/target/snappy-1.1.10/snappy.h b/target/snappy-1.1.10/snappy.h new file mode 100644 index 00000000..e9805bfb --- /dev/null +++ b/target/snappy-1.1.10/snappy.h @@ -0,0 +1,207 @@ +// Copyright 2005 and onwards Google Inc. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// A light-weight compression algorithm. It is designed for speed of +// compression and decompression, rather than for the utmost in space +// savings. +// +// For getting better compression ratios when you are compressing data +// with long repeated sequences or compressing data that is similar to +// other data, while still compressing fast, you might look at first +// using BMDiff and then compressing the output of BMDiff with +// Snappy. + +#ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__ +#define THIRD_PARTY_SNAPPY_SNAPPY_H__ + +#include +#include + +#include "snappy-stubs-public.h" + +namespace snappy { + class Source; + class Sink; + + // ------------------------------------------------------------------------ + // Generic compression/decompression routines. + // ------------------------------------------------------------------------ + + // Compress the bytes read from "*source" and append to "*sink". Return the + // number of bytes written. + size_t Compress(Source* source, Sink* sink); + + // Find the uncompressed length of the given stream, as given by the header. + // Note that the true length could deviate from this; the stream could e.g. + // be truncated. + // + // Also note that this leaves "*source" in a state that is unsuitable for + // further operations, such as RawUncompress(). You will need to rewind + // or recreate the source yourself before attempting any further calls. + bool GetUncompressedLength(Source* source, uint32* result); + + // ------------------------------------------------------------------------ + // Higher-level string based routines (should be sufficient for most users) + // ------------------------------------------------------------------------ + + // Sets "*compressed" to the compressed version of "input[0,input_length-1]". + // Original contents of *compressed are lost. + // + // REQUIRES: "input[]" is not an alias of "*compressed". + size_t Compress(const char* input, size_t input_length, + std::string* compressed); + + // Decompresses "compressed[0,compressed_length-1]" to "*uncompressed". + // Original contents of "*uncompressed" are lost. + // + // REQUIRES: "compressed[]" is not an alias of "*uncompressed". + // + // returns false if the message is corrupted and could not be decompressed + bool Uncompress(const char* compressed, size_t compressed_length, + std::string* uncompressed); + + // Decompresses "compressed" to "*uncompressed". + // + // returns false if the message is corrupted and could not be decompressed + bool Uncompress(Source* compressed, Sink* uncompressed); + + // This routine uncompresses as much of the "compressed" as possible + // into sink. It returns the number of valid bytes added to sink + // (extra invalid bytes may have been added due to errors; the caller + // should ignore those). The emitted data typically has length + // GetUncompressedLength(), but may be shorter if an error is + // encountered. + size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed); + + // ------------------------------------------------------------------------ + // Lower-level character array based routines. May be useful for + // efficiency reasons in certain circumstances. + // ------------------------------------------------------------------------ + + // REQUIRES: "compressed" must point to an area of memory that is at + // least "MaxCompressedLength(input_length)" bytes in length. + // + // Takes the data stored in "input[0..input_length]" and stores + // it in the array pointed to by "compressed". + // + // "*compressed_length" is set to the length of the compressed output. + // + // Example: + // char* output = new char[snappy::MaxCompressedLength(input_length)]; + // size_t output_length; + // RawCompress(input, input_length, output, &output_length); + // ... Process(output, output_length) ... + // delete [] output; + void RawCompress(const char* input, + size_t input_length, + char* compressed, + size_t* compressed_length); + + // Given data in "compressed[0..compressed_length-1]" generated by + // calling the Snappy::Compress routine, this routine + // stores the uncompressed data to + // uncompressed[0..GetUncompressedLength(compressed)-1] + // returns false if the message is corrupted and could not be decrypted + bool RawUncompress(const char* compressed, size_t compressed_length, + char* uncompressed); + + // Given data from the byte source 'compressed' generated by calling + // the Snappy::Compress routine, this routine stores the uncompressed + // data to + // uncompressed[0..GetUncompressedLength(compressed,compressed_length)-1] + // returns false if the message is corrupted and could not be decrypted + bool RawUncompress(Source* compressed, char* uncompressed); + + // Given data in "compressed[0..compressed_length-1]" generated by + // calling the Snappy::Compress routine, this routine + // stores the uncompressed data to the iovec "iov". The number of physical + // buffers in "iov" is given by iov_cnt and their cumulative size + // must be at least GetUncompressedLength(compressed). The individual buffers + // in "iov" must not overlap with each other. + // + // returns false if the message is corrupted and could not be decrypted + bool RawUncompressToIOVec(const char* compressed, size_t compressed_length, + const struct iovec* iov, size_t iov_cnt); + + // Given data from the byte source 'compressed' generated by calling + // the Snappy::Compress routine, this routine stores the uncompressed + // data to the iovec "iov". The number of physical + // buffers in "iov" is given by iov_cnt and their cumulative size + // must be at least GetUncompressedLength(compressed). The individual buffers + // in "iov" must not overlap with each other. + // + // returns false if the message is corrupted and could not be decrypted + bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov, + size_t iov_cnt); + + // Returns the maximal size of the compressed representation of + // input data that is "source_bytes" bytes in length; + size_t MaxCompressedLength(size_t source_bytes); + + // REQUIRES: "compressed[]" was produced by RawCompress() or Compress() + // Returns true and stores the length of the uncompressed data in + // *result normally. Returns false on parsing error. + // This operation takes O(1) time. + bool GetUncompressedLength(const char* compressed, size_t compressed_length, + size_t* result); + + // Returns true iff the contents of "compressed[]" can be uncompressed + // successfully. Does not return the uncompressed data. Takes + // time proportional to compressed_length, but is usually at least + // a factor of four faster than actual decompression. + bool IsValidCompressedBuffer(const char* compressed, + size_t compressed_length); + + // Returns true iff the contents of "compressed" can be uncompressed + // successfully. Does not return the uncompressed data. Takes + // time proportional to *compressed length, but is usually at least + // a factor of four faster than actual decompression. + // On success, consumes all of *compressed. On failure, consumes an + // unspecified prefix of *compressed. + bool IsValidCompressed(Source* compressed); + + // The size of a compression block. Note that many parts of the compression + // code assumes that kBlockSize <= 65536; in particular, the hash table + // can only store 16-bit offsets, and EmitCopy() also assumes the offset + // is 65535 bytes or less. Note also that if you change this, it will + // affect the framing format (see framing_format.txt). + // + // Note that there might be older data around that is compressed with larger + // block sizes, so the decompression code should not rely on the + // non-existence of long backreferences. + static constexpr int kBlockLog = 16; + static constexpr size_t kBlockSize = 1 << kBlockLog; + + static constexpr int kMinHashTableBits = 8; + static constexpr size_t kMinHashTableSize = 1 << kMinHashTableBits; + + static constexpr int kMaxHashTableBits = 14; + static constexpr size_t kMaxHashTableSize = 1 << kMaxHashTableBits; +} // end namespace snappy + +#endif // THIRD_PARTY_SNAPPY_SNAPPY_H__ diff --git a/target/snappy-1.1.10/snappy_compress_fuzzer.cc b/target/snappy-1.1.10/snappy_compress_fuzzer.cc new file mode 100644 index 00000000..1d0119e1 --- /dev/null +++ b/target/snappy-1.1.10/snappy_compress_fuzzer.cc @@ -0,0 +1,59 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// libFuzzer harness for fuzzing snappy compression code. + +#include +#include +#include +#include + +#include "snappy.h" + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + std::string input(reinterpret_cast(data), size); + + std::string compressed; + size_t compressed_size = + snappy::Compress(input.data(), input.size(), &compressed); + + (void)compressed_size; // Variable only used in debug builds. + assert(compressed_size == compressed.size()); + assert(compressed.size() <= snappy::MaxCompressedLength(input.size())); + assert(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + + std::string uncompressed_after_compress; + bool uncompress_succeeded = snappy::Uncompress( + compressed.data(), compressed.size(), &uncompressed_after_compress); + + (void)uncompress_succeeded; // Variable only used in debug builds. + assert(uncompress_succeeded); + assert(input == uncompressed_after_compress); + return 0; +} diff --git a/target/snappy-1.1.10/snappy_uncompress_fuzzer.cc b/target/snappy-1.1.10/snappy_uncompress_fuzzer.cc new file mode 100644 index 00000000..8071c00e --- /dev/null +++ b/target/snappy-1.1.10/snappy_uncompress_fuzzer.cc @@ -0,0 +1,57 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// libFuzzer harness for fuzzing snappy's decompression code. + +#include +#include +#include +#include + +#include "snappy.h" + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + std::string input(reinterpret_cast(data), size); + + // Avoid self-crafted decompression bombs. + size_t uncompressed_size; + constexpr size_t kMaxUncompressedSize = 1 << 20; + bool get_uncompressed_length_succeeded = snappy::GetUncompressedLength( + input.data(), input.size(), &uncompressed_size); + if (!get_uncompressed_length_succeeded || + (uncompressed_size > kMaxUncompressedSize)) { + return 0; + } + + std::string uncompressed; + // The return value of snappy::Uncompress() is ignored because decompression + // will fail on invalid inputs. + snappy::Uncompress(input.data(), input.size(), &uncompressed); + return 0; +} diff --git a/target/snappy-1.1.10/snappy_unittest.cc b/target/snappy-1.1.10/snappy_unittest.cc new file mode 100644 index 00000000..37159c32 --- /dev/null +++ b/target/snappy-1.1.10/snappy_unittest.cc @@ -0,0 +1,1506 @@ +// Copyright 2005 and onwards Google Inc. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include + +#include +#include +#include +#include +#include + +#include "snappy.h" +#include "snappy-internal.h" +#include "snappy-test.h" +#include "snappy-sinksource.h" + +DEFINE_int32(start_len, -1, + "Starting prefix size for testing (-1: just full file contents)"); +DEFINE_int32(end_len, -1, + "Starting prefix size for testing (-1: just full file contents)"); +DEFINE_int32(bytes, 10485760, + "How many bytes to compress/uncompress per file for timing"); + +DEFINE_bool(zlib, false, + "Run zlib compression (http://www.zlib.net)"); +DEFINE_bool(lzo, false, + "Run LZO compression (http://www.oberhumer.com/opensource/lzo/)"); +DEFINE_bool(snappy, true, "Run snappy compression"); + +DEFINE_bool(write_compressed, false, + "Write compressed versions of each file to .comp"); +DEFINE_bool(write_uncompressed, false, + "Write uncompressed versions of each file to .uncomp"); + +DEFINE_bool(snappy_dump_decompression_table, false, + "If true, we print the decompression table during tests."); + +namespace snappy { + +#if defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF) + +// To test against code that reads beyond its input, this class copies a +// string to a newly allocated group of pages, the last of which +// is made unreadable via mprotect. Note that we need to allocate the +// memory with mmap(), as POSIX allows mprotect() only on memory allocated +// with mmap(), and some malloc/posix_memalign implementations expect to +// be able to read previously allocated memory while doing heap allocations. +class DataEndingAtUnreadablePage { + public: + explicit DataEndingAtUnreadablePage(const std::string& s) { + const size_t page_size = sysconf(_SC_PAGESIZE); + const size_t size = s.size(); + // Round up space for string to a multiple of page_size. + size_t space_for_string = (size + page_size - 1) & ~(page_size - 1); + alloc_size_ = space_for_string + page_size; + mem_ = mmap(NULL, alloc_size_, + PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + CHECK_NE(MAP_FAILED, mem_); + protected_page_ = reinterpret_cast(mem_) + space_for_string; + char* dst = protected_page_ - size; + memcpy(dst, s.data(), size); + data_ = dst; + size_ = size; + // Make guard page unreadable. + CHECK_EQ(0, mprotect(protected_page_, page_size, PROT_NONE)); + } + + ~DataEndingAtUnreadablePage() { + const size_t page_size = sysconf(_SC_PAGESIZE); + // Undo the mprotect. + CHECK_EQ(0, mprotect(protected_page_, page_size, PROT_READ|PROT_WRITE)); + CHECK_EQ(0, munmap(mem_, alloc_size_)); + } + + const char* data() const { return data_; } + size_t size() const { return size_; } + + private: + size_t alloc_size_; + void* mem_; + char* protected_page_; + const char* data_; + size_t size_; +}; + +#else // defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF) + +// Fallback for systems without mmap. +using DataEndingAtUnreadablePage = std::string; + +#endif + +enum CompressorType { + ZLIB, LZO, SNAPPY +}; + +const char* names[] = { + "ZLIB", "LZO", "SNAPPY" +}; + +static size_t MinimumRequiredOutputSpace(size_t input_size, + CompressorType comp) { + switch (comp) { +#ifdef ZLIB_VERSION + case ZLIB: + return ZLib::MinCompressbufSize(input_size); +#endif // ZLIB_VERSION + +#ifdef LZO_VERSION + case LZO: + return input_size + input_size/64 + 16 + 3; +#endif // LZO_VERSION + + case SNAPPY: + return snappy::MaxCompressedLength(input_size); + + default: + LOG(FATAL) << "Unknown compression type number " << comp; + return 0; + } +} + +// Returns true if we successfully compressed, false otherwise. +// +// If compressed_is_preallocated is set, do not resize the compressed buffer. +// This is typically what you want for a benchmark, in order to not spend +// time in the memory allocator. If you do set this flag, however, +// "compressed" must be preinitialized to at least MinCompressbufSize(comp) +// number of bytes, and may contain junk bytes at the end after return. +static bool Compress(const char* input, size_t input_size, CompressorType comp, + std::string* compressed, bool compressed_is_preallocated) { + if (!compressed_is_preallocated) { + compressed->resize(MinimumRequiredOutputSpace(input_size, comp)); + } + + switch (comp) { +#ifdef ZLIB_VERSION + case ZLIB: { + ZLib zlib; + uLongf destlen = compressed->size(); + int ret = zlib.Compress( + reinterpret_cast(string_as_array(compressed)), + &destlen, + reinterpret_cast(input), + input_size); + CHECK_EQ(Z_OK, ret); + if (!compressed_is_preallocated) { + compressed->resize(destlen); + } + return true; + } +#endif // ZLIB_VERSION + +#ifdef LZO_VERSION + case LZO: { + unsigned char* mem = new unsigned char[LZO1X_1_15_MEM_COMPRESS]; + lzo_uint destlen; + int ret = lzo1x_1_15_compress( + reinterpret_cast(input), + input_size, + reinterpret_cast(string_as_array(compressed)), + &destlen, + mem); + CHECK_EQ(LZO_E_OK, ret); + delete[] mem; + if (!compressed_is_preallocated) { + compressed->resize(destlen); + } + break; + } +#endif // LZO_VERSION + + case SNAPPY: { + size_t destlen; + snappy::RawCompress(input, input_size, + string_as_array(compressed), + &destlen); + CHECK_LE(destlen, snappy::MaxCompressedLength(input_size)); + if (!compressed_is_preallocated) { + compressed->resize(destlen); + } + break; + } + + default: { + return false; // the asked-for library wasn't compiled in + } + } + return true; +} + +static bool Uncompress(const std::string& compressed, CompressorType comp, + int size, std::string* output) { + switch (comp) { +#ifdef ZLIB_VERSION + case ZLIB: { + output->resize(size); + ZLib zlib; + uLongf destlen = output->size(); + int ret = zlib.Uncompress( + reinterpret_cast(string_as_array(output)), + &destlen, + reinterpret_cast(compressed.data()), + compressed.size()); + CHECK_EQ(Z_OK, ret); + CHECK_EQ(static_cast(size), destlen); + break; + } +#endif // ZLIB_VERSION + +#ifdef LZO_VERSION + case LZO: { + output->resize(size); + lzo_uint destlen; + int ret = lzo1x_decompress( + reinterpret_cast(compressed.data()), + compressed.size(), + reinterpret_cast(string_as_array(output)), + &destlen, + NULL); + CHECK_EQ(LZO_E_OK, ret); + CHECK_EQ(static_cast(size), destlen); + break; + } +#endif // LZO_VERSION + + case SNAPPY: { + snappy::RawUncompress(compressed.data(), compressed.size(), + string_as_array(output)); + break; + } + + default: { + return false; // the asked-for library wasn't compiled in + } + } + return true; +} + +static void Measure(const char* data, + size_t length, + CompressorType comp, + int repeats, + int block_size) { + // Run tests a few time and pick median running times + static const int kRuns = 5; + double ctime[kRuns]; + double utime[kRuns]; + int compressed_size = 0; + + { + // Chop the input into blocks + int num_blocks = (length + block_size - 1) / block_size; + std::vector input(num_blocks); + std::vector input_length(num_blocks); + std::vector compressed(num_blocks); + std::vector output(num_blocks); + for (int b = 0; b < num_blocks; b++) { + int input_start = b * block_size; + int input_limit = std::min((b+1)*block_size, length); + input[b] = data+input_start; + input_length[b] = input_limit-input_start; + + // Pre-grow the output buffer so we don't measure string append time. + compressed[b].resize(MinimumRequiredOutputSpace(block_size, comp)); + } + + // First, try one trial compression to make sure the code is compiled in + if (!Compress(input[0], input_length[0], comp, &compressed[0], true)) { + LOG(WARNING) << "Skipping " << names[comp] << ": " + << "library not compiled in"; + return; + } + + for (int run = 0; run < kRuns; run++) { + CycleTimer ctimer, utimer; + + for (int b = 0; b < num_blocks; b++) { + // Pre-grow the output buffer so we don't measure string append time. + compressed[b].resize(MinimumRequiredOutputSpace(block_size, comp)); + } + + ctimer.Start(); + for (int b = 0; b < num_blocks; b++) + for (int i = 0; i < repeats; i++) + Compress(input[b], input_length[b], comp, &compressed[b], true); + ctimer.Stop(); + + // Compress once more, with resizing, so we don't leave junk + // at the end that will confuse the decompressor. + for (int b = 0; b < num_blocks; b++) { + Compress(input[b], input_length[b], comp, &compressed[b], false); + } + + for (int b = 0; b < num_blocks; b++) { + output[b].resize(input_length[b]); + } + + utimer.Start(); + for (int i = 0; i < repeats; i++) + for (int b = 0; b < num_blocks; b++) + Uncompress(compressed[b], comp, input_length[b], &output[b]); + utimer.Stop(); + + ctime[run] = ctimer.Get(); + utime[run] = utimer.Get(); + } + + compressed_size = 0; + for (size_t i = 0; i < compressed.size(); i++) { + compressed_size += compressed[i].size(); + } + } + + std::sort(ctime, ctime + kRuns); + std::sort(utime, utime + kRuns); + const int med = kRuns/2; + + float comp_rate = (length / ctime[med]) * repeats / 1048576.0; + float uncomp_rate = (length / utime[med]) * repeats / 1048576.0; + std::string x = names[comp]; + x += ":"; + std::string urate = (uncomp_rate >= 0) ? StrFormat("%.1f", uncomp_rate) + : std::string("?"); + printf("%-7s [b %dM] bytes %6d -> %6d %4.1f%% " + "comp %5.1f MB/s uncomp %5s MB/s\n", + x.c_str(), + block_size/(1<<20), + static_cast(length), static_cast(compressed_size), + (compressed_size * 100.0) / std::max(1, length), + comp_rate, + urate.c_str()); +} + +static int VerifyString(const std::string& input) { + std::string compressed; + DataEndingAtUnreadablePage i(input); + const size_t written = snappy::Compress(i.data(), i.size(), &compressed); + CHECK_EQ(written, compressed.size()); + CHECK_LE(compressed.size(), + snappy::MaxCompressedLength(input.size())); + CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + + std::string uncompressed; + DataEndingAtUnreadablePage c(compressed); + CHECK(snappy::Uncompress(c.data(), c.size(), &uncompressed)); + CHECK_EQ(uncompressed, input); + return uncompressed.size(); +} + +static void VerifyStringSink(const std::string& input) { + std::string compressed; + DataEndingAtUnreadablePage i(input); + const size_t written = snappy::Compress(i.data(), i.size(), &compressed); + CHECK_EQ(written, compressed.size()); + CHECK_LE(compressed.size(), + snappy::MaxCompressedLength(input.size())); + CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + + std::string uncompressed; + uncompressed.resize(input.size()); + snappy::UncheckedByteArraySink sink(string_as_array(&uncompressed)); + DataEndingAtUnreadablePage c(compressed); + snappy::ByteArraySource source(c.data(), c.size()); + CHECK(snappy::Uncompress(&source, &sink)); + CHECK_EQ(uncompressed, input); +} + +static void VerifyIOVec(const std::string& input) { + std::string compressed; + DataEndingAtUnreadablePage i(input); + const size_t written = snappy::Compress(i.data(), i.size(), &compressed); + CHECK_EQ(written, compressed.size()); + CHECK_LE(compressed.size(), + snappy::MaxCompressedLength(input.size())); + CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + + // Try uncompressing into an iovec containing a random number of entries + // ranging from 1 to 10. + char* buf = new char[input.size()]; + std::minstd_rand0 rng(input.size()); + std::uniform_int_distribution uniform_1_to_10(1, 10); + size_t num = uniform_1_to_10(rng); + if (input.size() < num) { + num = input.size(); + } + struct iovec* iov = new iovec[num]; + int used_so_far = 0; + std::bernoulli_distribution one_in_five(1.0 / 5); + for (size_t i = 0; i < num; ++i) { + assert(used_so_far < input.size()); + iov[i].iov_base = buf + used_so_far; + if (i == num - 1) { + iov[i].iov_len = input.size() - used_so_far; + } else { + // Randomly choose to insert a 0 byte entry. + if (one_in_five(rng)) { + iov[i].iov_len = 0; + } else { + std::uniform_int_distribution uniform_not_used_so_far( + 0, input.size() - used_so_far - 1); + iov[i].iov_len = uniform_not_used_so_far(rng); + } + } + used_so_far += iov[i].iov_len; + } + CHECK(snappy::RawUncompressToIOVec( + compressed.data(), compressed.size(), iov, num)); + CHECK(!memcmp(buf, input.data(), input.size())); + delete[] iov; + delete[] buf; +} + +// Test that data compressed by a compressor that does not +// obey block sizes is uncompressed properly. +static void VerifyNonBlockedCompression(const std::string& input) { + if (input.length() > snappy::kBlockSize) { + // We cannot test larger blocks than the maximum block size, obviously. + return; + } + + std::string prefix; + Varint::Append32(&prefix, input.size()); + + // Setup compression table + snappy::internal::WorkingMemory wmem(input.size()); + int table_size; + uint16* table = wmem.GetHashTable(input.size(), &table_size); + + // Compress entire input in one shot + std::string compressed; + compressed += prefix; + compressed.resize(prefix.size()+snappy::MaxCompressedLength(input.size())); + char* dest = string_as_array(&compressed) + prefix.size(); + char* end = snappy::internal::CompressFragment(input.data(), input.size(), + dest, table, table_size); + compressed.resize(end - compressed.data()); + + // Uncompress into std::string + std::string uncomp_str; + CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncomp_str)); + CHECK_EQ(uncomp_str, input); + + // Uncompress using source/sink + std::string uncomp_str2; + uncomp_str2.resize(input.size()); + snappy::UncheckedByteArraySink sink(string_as_array(&uncomp_str2)); + snappy::ByteArraySource source(compressed.data(), compressed.size()); + CHECK(snappy::Uncompress(&source, &sink)); + CHECK_EQ(uncomp_str2, input); + + // Uncompress into iovec + { + static const int kNumBlocks = 10; + struct iovec vec[kNumBlocks]; + const int block_size = 1 + input.size() / kNumBlocks; + std::string iovec_data(block_size * kNumBlocks, 'x'); + for (int i = 0; i < kNumBlocks; i++) { + vec[i].iov_base = string_as_array(&iovec_data) + i * block_size; + vec[i].iov_len = block_size; + } + CHECK(snappy::RawUncompressToIOVec(compressed.data(), compressed.size(), + vec, kNumBlocks)); + CHECK_EQ(std::string(iovec_data.data(), input.size()), input); + } +} + +// Expand the input so that it is at least K times as big as block size +static std::string Expand(const std::string& input) { + static const int K = 3; + std::string data = input; + while (data.size() < K * snappy::kBlockSize) { + data += input; + } + return data; +} + +static int Verify(const std::string& input) { + VLOG(1) << "Verifying input of size " << input.size(); + + // Compress using string based routines + const int result = VerifyString(input); + + // Verify using sink based routines + VerifyStringSink(input); + + VerifyNonBlockedCompression(input); + VerifyIOVec(input); + if (!input.empty()) { + const std::string expanded = Expand(input); + VerifyNonBlockedCompression(expanded); + VerifyIOVec(input); + } + + return result; +} + +static bool IsValidCompressedBuffer(const std::string& c) { + return snappy::IsValidCompressedBuffer(c.data(), c.size()); +} +static bool Uncompress(const std::string& c, std::string* u) { + return snappy::Uncompress(c.data(), c.size(), u); +} + +// This test checks to ensure that snappy doesn't coredump if it gets +// corrupted data. +TEST(CorruptedTest, VerifyCorrupted) { + std::string source = "making sure we don't crash with corrupted input"; + VLOG(1) << source; + std::string dest; + std::string uncmp; + snappy::Compress(source.data(), source.size(), &dest); + + // Mess around with the data. It's hard to simulate all possible + // corruptions; this is just one example ... + CHECK_GT(dest.size(), 3); + dest[1]--; + dest[3]++; + // this really ought to fail. + CHECK(!IsValidCompressedBuffer(dest)); + CHECK(!Uncompress(dest, &uncmp)); + + // This is testing for a security bug - a buffer that decompresses to 100k + // but we lie in the snappy header and only reserve 0 bytes of memory :) + source.resize(100000); + for (size_t i = 0; i < source.length(); ++i) { + source[i] = 'A'; + } + snappy::Compress(source.data(), source.size(), &dest); + dest[0] = dest[1] = dest[2] = dest[3] = 0; + CHECK(!IsValidCompressedBuffer(dest)); + CHECK(!Uncompress(dest, &uncmp)); + + if (sizeof(void *) == 4) { + // Another security check; check a crazy big length can't DoS us with an + // over-allocation. + // Currently this is done only for 32-bit builds. On 64-bit builds, + // where 3 GB might be an acceptable allocation size, Uncompress() + // attempts to decompress, and sometimes causes the test to run out of + // memory. + dest[0] = dest[1] = dest[2] = dest[3] = '\xff'; + // This decodes to a really large size, i.e., about 3 GB. + dest[4] = 'k'; + CHECK(!IsValidCompressedBuffer(dest)); + CHECK(!Uncompress(dest, &uncmp)); + } else { + LOG(WARNING) << "Crazy decompression lengths not checked on 64-bit build"; + } + + // This decodes to about 2 MB; much smaller, but should still fail. + dest[0] = dest[1] = dest[2] = '\xff'; + dest[3] = 0x00; + CHECK(!IsValidCompressedBuffer(dest)); + CHECK(!Uncompress(dest, &uncmp)); + + // try reading stuff in from a bad file. + for (int i = 1; i <= 3; ++i) { + std::string data = + ReadTestDataFile(StrFormat("baddata%d.snappy", i).c_str(), 0); + std::string uncmp; + // check that we don't return a crazy length + size_t ulen; + CHECK(!snappy::GetUncompressedLength(data.data(), data.size(), &ulen) + || (ulen < (1<<20))); + uint32 ulen2; + snappy::ByteArraySource source(data.data(), data.size()); + CHECK(!snappy::GetUncompressedLength(&source, &ulen2) || + (ulen2 < (1<<20))); + CHECK(!IsValidCompressedBuffer(data)); + CHECK(!Uncompress(data, &uncmp)); + } +} + +// Helper routines to construct arbitrary compressed strings. +// These mirror the compression code in snappy.cc, but are copied +// here so that we can bypass some limitations in the how snappy.cc +// invokes these routines. +static void AppendLiteral(std::string* dst, const std::string& literal) { + if (literal.empty()) return; + int n = literal.size() - 1; + if (n < 60) { + // Fit length in tag byte + dst->push_back(0 | (n << 2)); + } else { + // Encode in upcoming bytes + char number[4]; + int count = 0; + while (n > 0) { + number[count++] = n & 0xff; + n >>= 8; + } + dst->push_back(0 | ((59+count) << 2)); + *dst += std::string(number, count); + } + *dst += literal; +} + +static void AppendCopy(std::string* dst, int offset, int length) { + while (length > 0) { + // Figure out how much to copy in one shot + int to_copy; + if (length >= 68) { + to_copy = 64; + } else if (length > 64) { + to_copy = 60; + } else { + to_copy = length; + } + length -= to_copy; + + if ((to_copy >= 4) && (to_copy < 12) && (offset < 2048)) { + assert(to_copy-4 < 8); // Must fit in 3 bits + dst->push_back(1 | ((to_copy-4) << 2) | ((offset >> 8) << 5)); + dst->push_back(offset & 0xff); + } else if (offset < 65536) { + dst->push_back(2 | ((to_copy-1) << 2)); + dst->push_back(offset & 0xff); + dst->push_back(offset >> 8); + } else { + dst->push_back(3 | ((to_copy-1) << 2)); + dst->push_back(offset & 0xff); + dst->push_back((offset >> 8) & 0xff); + dst->push_back((offset >> 16) & 0xff); + dst->push_back((offset >> 24) & 0xff); + } + } +} + +TEST(Snappy, SimpleTests) { + Verify(""); + Verify("a"); + Verify("ab"); + Verify("abc"); + + Verify("aaaaaaa" + std::string(16, 'b') + std::string("aaaaa") + "abc"); + Verify("aaaaaaa" + std::string(256, 'b') + std::string("aaaaa") + "abc"); + Verify("aaaaaaa" + std::string(2047, 'b') + std::string("aaaaa") + "abc"); + Verify("aaaaaaa" + std::string(65536, 'b') + std::string("aaaaa") + "abc"); + Verify("abcaaaaaaa" + std::string(65536, 'b') + std::string("aaaaa") + "abc"); +} + +// Verify max blowup (lots of four-byte copies) +TEST(Snappy, MaxBlowup) { + std::mt19937 rng; + std::uniform_int_distribution uniform_byte(0, 255); + std::string input; + for (int i = 0; i < 80000; ++i) + input.push_back(static_cast(uniform_byte(rng))); + + for (int i = 0; i < 80000; i += 4) { + std::string four_bytes(input.end() - i - 4, input.end() - i); + input.append(four_bytes); + } + Verify(input); +} + +TEST(Snappy, RandomData) { + std::minstd_rand0 rng(FLAGS_test_random_seed); + std::uniform_int_distribution uniform_0_to_3(0, 3); + std::uniform_int_distribution uniform_0_to_8(0, 8); + std::uniform_int_distribution uniform_byte(0, 255); + std::uniform_int_distribution uniform_4k(0, 4095); + std::uniform_int_distribution uniform_64k(0, 65535); + std::bernoulli_distribution one_in_ten(1.0 / 10); + + constexpr int num_ops = 20000; + for (int i = 0; i < num_ops; i++) { + if ((i % 1000) == 0) { + VLOG(0) << "Random op " << i << " of " << num_ops; + } + + std::string x; + size_t len = uniform_4k(rng); + if (i < 100) { + len = 65536 + uniform_64k(rng); + } + while (x.size() < len) { + int run_len = 1; + if (one_in_ten(rng)) { + int skewed_bits = uniform_0_to_8(rng); + // int is guaranteed to hold at least 16 bits, this uses at most 8 bits. + std::uniform_int_distribution skewed_low(0, + (1 << skewed_bits) - 1); + run_len = skewed_low(rng); + } + char c = static_cast(uniform_byte(rng)); + if (i >= 100) { + int skewed_bits = uniform_0_to_3(rng); + // int is guaranteed to hold at least 16 bits, this uses at most 3 bits. + std::uniform_int_distribution skewed_low(0, + (1 << skewed_bits) - 1); + c = static_cast(skewed_low(rng)); + } + while (run_len-- > 0 && x.size() < len) { + x.push_back(c); + } + } + + Verify(x); + } +} + +TEST(Snappy, FourByteOffset) { + // The new compressor cannot generate four-byte offsets since + // it chops up the input into 32KB pieces. So we hand-emit the + // copy manually. + + // The two fragments that make up the input string. + std::string fragment1 = "012345689abcdefghijklmnopqrstuvwxyz"; + std::string fragment2 = "some other string"; + + // How many times each fragment is emitted. + const int n1 = 2; + const int n2 = 100000 / fragment2.size(); + const int length = n1 * fragment1.size() + n2 * fragment2.size(); + + std::string compressed; + Varint::Append32(&compressed, length); + + AppendLiteral(&compressed, fragment1); + std::string src = fragment1; + for (int i = 0; i < n2; i++) { + AppendLiteral(&compressed, fragment2); + src += fragment2; + } + AppendCopy(&compressed, src.size(), fragment1.size()); + src += fragment1; + CHECK_EQ(length, src.size()); + + std::string uncompressed; + CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + CHECK(snappy::Uncompress(compressed.data(), compressed.size(), + &uncompressed)); + CHECK_EQ(uncompressed, src); +} + +TEST(Snappy, IOVecEdgeCases) { + // Test some tricky edge cases in the iovec output that are not necessarily + // exercised by random tests. + + // Our output blocks look like this initially (the last iovec is bigger + // than depicted): + // [ ] [ ] [ ] [ ] [ ] + static const int kLengths[] = { 2, 1, 4, 8, 128 }; + + struct iovec iov[ARRAYSIZE(kLengths)]; + for (int i = 0; i < ARRAYSIZE(kLengths); ++i) { + iov[i].iov_base = new char[kLengths[i]]; + iov[i].iov_len = kLengths[i]; + } + + std::string compressed; + Varint::Append32(&compressed, 22); + + // A literal whose output crosses three blocks. + // [ab] [c] [123 ] [ ] [ ] + AppendLiteral(&compressed, "abc123"); + + // A copy whose output crosses two blocks (source and destination + // segments marked). + // [ab] [c] [1231] [23 ] [ ] + // ^--^ -- + AppendCopy(&compressed, 3, 3); + + // A copy where the input is, at first, in the block before the output: + // + // [ab] [c] [1231] [231231 ] [ ] + // ^--- ^--- + // Then during the copy, the pointers move such that the input and + // output pointers are in the same block: + // + // [ab] [c] [1231] [23123123] [ ] + // ^- ^- + // And then they move again, so that the output pointer is no longer + // in the same block as the input pointer: + // [ab] [c] [1231] [23123123] [123 ] + // ^-- ^-- + AppendCopy(&compressed, 6, 9); + + // Finally, a copy where the input is from several blocks back, + // and it also crosses three blocks: + // + // [ab] [c] [1231] [23123123] [123b ] + // ^ ^ + // [ab] [c] [1231] [23123123] [123bc ] + // ^ ^ + // [ab] [c] [1231] [23123123] [123bc12 ] + // ^- ^- + AppendCopy(&compressed, 17, 4); + + CHECK(snappy::RawUncompressToIOVec( + compressed.data(), compressed.size(), iov, ARRAYSIZE(iov))); + CHECK_EQ(0, memcmp(iov[0].iov_base, "ab", 2)); + CHECK_EQ(0, memcmp(iov[1].iov_base, "c", 1)); + CHECK_EQ(0, memcmp(iov[2].iov_base, "1231", 4)); + CHECK_EQ(0, memcmp(iov[3].iov_base, "23123123", 8)); + CHECK_EQ(0, memcmp(iov[4].iov_base, "123bc12", 7)); + + for (int i = 0; i < ARRAYSIZE(kLengths); ++i) { + delete[] reinterpret_cast(iov[i].iov_base); + } +} + +TEST(Snappy, IOVecLiteralOverflow) { + static const int kLengths[] = { 3, 4 }; + + struct iovec iov[ARRAYSIZE(kLengths)]; + for (int i = 0; i < ARRAYSIZE(kLengths); ++i) { + iov[i].iov_base = new char[kLengths[i]]; + iov[i].iov_len = kLengths[i]; + } + + std::string compressed; + Varint::Append32(&compressed, 8); + + AppendLiteral(&compressed, "12345678"); + + CHECK(!snappy::RawUncompressToIOVec( + compressed.data(), compressed.size(), iov, ARRAYSIZE(iov))); + + for (int i = 0; i < ARRAYSIZE(kLengths); ++i) { + delete[] reinterpret_cast(iov[i].iov_base); + } +} + +TEST(Snappy, IOVecCopyOverflow) { + static const int kLengths[] = { 3, 4 }; + + struct iovec iov[ARRAYSIZE(kLengths)]; + for (int i = 0; i < ARRAYSIZE(kLengths); ++i) { + iov[i].iov_base = new char[kLengths[i]]; + iov[i].iov_len = kLengths[i]; + } + + std::string compressed; + Varint::Append32(&compressed, 8); + + AppendLiteral(&compressed, "123"); + AppendCopy(&compressed, 3, 5); + + CHECK(!snappy::RawUncompressToIOVec( + compressed.data(), compressed.size(), iov, ARRAYSIZE(iov))); + + for (int i = 0; i < ARRAYSIZE(kLengths); ++i) { + delete[] reinterpret_cast(iov[i].iov_base); + } +} + +static bool CheckUncompressedLength(const std::string& compressed, + size_t* ulength) { + const bool result1 = snappy::GetUncompressedLength(compressed.data(), + compressed.size(), + ulength); + + snappy::ByteArraySource source(compressed.data(), compressed.size()); + uint32 length; + const bool result2 = snappy::GetUncompressedLength(&source, &length); + CHECK_EQ(result1, result2); + return result1; +} + +TEST(SnappyCorruption, TruncatedVarint) { + std::string compressed, uncompressed; + size_t ulength; + compressed.push_back('\xf0'); + CHECK(!CheckUncompressedLength(compressed, &ulength)); + CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + CHECK(!snappy::Uncompress(compressed.data(), compressed.size(), + &uncompressed)); +} + +TEST(SnappyCorruption, UnterminatedVarint) { + std::string compressed, uncompressed; + size_t ulength; + compressed.push_back('\x80'); + compressed.push_back('\x80'); + compressed.push_back('\x80'); + compressed.push_back('\x80'); + compressed.push_back('\x80'); + compressed.push_back(10); + CHECK(!CheckUncompressedLength(compressed, &ulength)); + CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + CHECK(!snappy::Uncompress(compressed.data(), compressed.size(), + &uncompressed)); +} + +TEST(SnappyCorruption, OverflowingVarint) { + std::string compressed, uncompressed; + size_t ulength; + compressed.push_back('\xfb'); + compressed.push_back('\xff'); + compressed.push_back('\xff'); + compressed.push_back('\xff'); + compressed.push_back('\x7f'); + CHECK(!CheckUncompressedLength(compressed, &ulength)); + CHECK(!snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + CHECK(!snappy::Uncompress(compressed.data(), compressed.size(), + &uncompressed)); +} + +TEST(Snappy, ReadPastEndOfBuffer) { + // Check that we do not read past end of input + + // Make a compressed string that ends with a single-byte literal + std::string compressed; + Varint::Append32(&compressed, 1); + AppendLiteral(&compressed, "x"); + + std::string uncompressed; + DataEndingAtUnreadablePage c(compressed); + CHECK(snappy::Uncompress(c.data(), c.size(), &uncompressed)); + CHECK_EQ(uncompressed, std::string("x")); +} + +// Check for an infinite loop caused by a copy with offset==0 +TEST(Snappy, ZeroOffsetCopy) { + const char* compressed = "\x40\x12\x00\x00"; + // \x40 Length (must be > kMaxIncrementCopyOverflow) + // \x12\x00\x00 Copy with offset==0, length==5 + char uncompressed[100]; + EXPECT_FALSE(snappy::RawUncompress(compressed, 4, uncompressed)); +} + +TEST(Snappy, ZeroOffsetCopyValidation) { + const char* compressed = "\x05\x12\x00\x00"; + // \x05 Length + // \x12\x00\x00 Copy with offset==0, length==5 + EXPECT_FALSE(snappy::IsValidCompressedBuffer(compressed, 4)); +} + +namespace { + +int TestFindMatchLength(const char* s1, const char *s2, unsigned length) { + std::pair p = + snappy::internal::FindMatchLength(s1, s2, s2 + length); + CHECK_EQ(p.first < 8, p.second); + return p.first; +} + +} // namespace + +TEST(Snappy, FindMatchLength) { + // Exercise all different code paths through the function. + // 64-bit version: + + // Hit s1_limit in 64-bit loop, hit s1_limit in single-character loop. + EXPECT_EQ(6, TestFindMatchLength("012345", "012345", 6)); + EXPECT_EQ(11, TestFindMatchLength("01234567abc", "01234567abc", 11)); + + // Hit s1_limit in 64-bit loop, find a non-match in single-character loop. + EXPECT_EQ(9, TestFindMatchLength("01234567abc", "01234567axc", 9)); + + // Same, but edge cases. + EXPECT_EQ(11, TestFindMatchLength("01234567abc!", "01234567abc!", 11)); + EXPECT_EQ(11, TestFindMatchLength("01234567abc!", "01234567abc?", 11)); + + // Find non-match at once in first loop. + EXPECT_EQ(0, TestFindMatchLength("01234567xxxxxxxx", "?1234567xxxxxxxx", 16)); + EXPECT_EQ(1, TestFindMatchLength("01234567xxxxxxxx", "0?234567xxxxxxxx", 16)); + EXPECT_EQ(4, TestFindMatchLength("01234567xxxxxxxx", "01237654xxxxxxxx", 16)); + EXPECT_EQ(7, TestFindMatchLength("01234567xxxxxxxx", "0123456?xxxxxxxx", 16)); + + // Find non-match in first loop after one block. + EXPECT_EQ(8, TestFindMatchLength("abcdefgh01234567xxxxxxxx", + "abcdefgh?1234567xxxxxxxx", 24)); + EXPECT_EQ(9, TestFindMatchLength("abcdefgh01234567xxxxxxxx", + "abcdefgh0?234567xxxxxxxx", 24)); + EXPECT_EQ(12, TestFindMatchLength("abcdefgh01234567xxxxxxxx", + "abcdefgh01237654xxxxxxxx", 24)); + EXPECT_EQ(15, TestFindMatchLength("abcdefgh01234567xxxxxxxx", + "abcdefgh0123456?xxxxxxxx", 24)); + + // 32-bit version: + + // Short matches. + EXPECT_EQ(0, TestFindMatchLength("01234567", "?1234567", 8)); + EXPECT_EQ(1, TestFindMatchLength("01234567", "0?234567", 8)); + EXPECT_EQ(2, TestFindMatchLength("01234567", "01?34567", 8)); + EXPECT_EQ(3, TestFindMatchLength("01234567", "012?4567", 8)); + EXPECT_EQ(4, TestFindMatchLength("01234567", "0123?567", 8)); + EXPECT_EQ(5, TestFindMatchLength("01234567", "01234?67", 8)); + EXPECT_EQ(6, TestFindMatchLength("01234567", "012345?7", 8)); + EXPECT_EQ(7, TestFindMatchLength("01234567", "0123456?", 8)); + EXPECT_EQ(7, TestFindMatchLength("01234567", "0123456?", 7)); + EXPECT_EQ(7, TestFindMatchLength("01234567!", "0123456??", 7)); + + // Hit s1_limit in 32-bit loop, hit s1_limit in single-character loop. + EXPECT_EQ(10, TestFindMatchLength("xxxxxxabcd", "xxxxxxabcd", 10)); + EXPECT_EQ(10, TestFindMatchLength("xxxxxxabcd?", "xxxxxxabcd?", 10)); + EXPECT_EQ(13, TestFindMatchLength("xxxxxxabcdef", "xxxxxxabcdef", 13)); + + // Same, but edge cases. + EXPECT_EQ(12, TestFindMatchLength("xxxxxx0123abc!", "xxxxxx0123abc!", 12)); + EXPECT_EQ(12, TestFindMatchLength("xxxxxx0123abc!", "xxxxxx0123abc?", 12)); + + // Hit s1_limit in 32-bit loop, find a non-match in single-character loop. + EXPECT_EQ(11, TestFindMatchLength("xxxxxx0123abc", "xxxxxx0123axc", 13)); + + // Find non-match at once in first loop. + EXPECT_EQ(6, TestFindMatchLength("xxxxxx0123xxxxxxxx", + "xxxxxx?123xxxxxxxx", 18)); + EXPECT_EQ(7, TestFindMatchLength("xxxxxx0123xxxxxxxx", + "xxxxxx0?23xxxxxxxx", 18)); + EXPECT_EQ(8, TestFindMatchLength("xxxxxx0123xxxxxxxx", + "xxxxxx0132xxxxxxxx", 18)); + EXPECT_EQ(9, TestFindMatchLength("xxxxxx0123xxxxxxxx", + "xxxxxx012?xxxxxxxx", 18)); + + // Same, but edge cases. + EXPECT_EQ(6, TestFindMatchLength("xxxxxx0123", "xxxxxx?123", 10)); + EXPECT_EQ(7, TestFindMatchLength("xxxxxx0123", "xxxxxx0?23", 10)); + EXPECT_EQ(8, TestFindMatchLength("xxxxxx0123", "xxxxxx0132", 10)); + EXPECT_EQ(9, TestFindMatchLength("xxxxxx0123", "xxxxxx012?", 10)); + + // Find non-match in first loop after one block. + EXPECT_EQ(10, TestFindMatchLength("xxxxxxabcd0123xx", + "xxxxxxabcd?123xx", 16)); + EXPECT_EQ(11, TestFindMatchLength("xxxxxxabcd0123xx", + "xxxxxxabcd0?23xx", 16)); + EXPECT_EQ(12, TestFindMatchLength("xxxxxxabcd0123xx", + "xxxxxxabcd0132xx", 16)); + EXPECT_EQ(13, TestFindMatchLength("xxxxxxabcd0123xx", + "xxxxxxabcd012?xx", 16)); + + // Same, but edge cases. + EXPECT_EQ(10, TestFindMatchLength("xxxxxxabcd0123", "xxxxxxabcd?123", 14)); + EXPECT_EQ(11, TestFindMatchLength("xxxxxxabcd0123", "xxxxxxabcd0?23", 14)); + EXPECT_EQ(12, TestFindMatchLength("xxxxxxabcd0123", "xxxxxxabcd0132", 14)); + EXPECT_EQ(13, TestFindMatchLength("xxxxxxabcd0123", "xxxxxxabcd012?", 14)); +} + +TEST(Snappy, FindMatchLengthRandom) { + constexpr int kNumTrials = 10000; + constexpr int kTypicalLength = 10; + std::minstd_rand0 rng(FLAGS_test_random_seed); + std::uniform_int_distribution uniform_byte(0, 255); + std::bernoulli_distribution one_in_two(1.0 / 2); + std::bernoulli_distribution one_in_typical_length(1.0 / kTypicalLength); + + for (int i = 0; i < kNumTrials; i++) { + std::string s, t; + char a = static_cast(uniform_byte(rng)); + char b = static_cast(uniform_byte(rng)); + while (!one_in_typical_length(rng)) { + s.push_back(one_in_two(rng) ? a : b); + t.push_back(one_in_two(rng) ? a : b); + } + DataEndingAtUnreadablePage u(s); + DataEndingAtUnreadablePage v(t); + int matched = TestFindMatchLength(u.data(), v.data(), t.size()); + if (matched == t.size()) { + EXPECT_EQ(s, t); + } else { + EXPECT_NE(s[matched], t[matched]); + for (int j = 0; j < matched; j++) { + EXPECT_EQ(s[j], t[j]); + } + } + } +} + +static uint16 MakeEntry(unsigned int extra, + unsigned int len, + unsigned int copy_offset) { + // Check that all of the fields fit within the allocated space + assert(extra == (extra & 0x7)); // At most 3 bits + assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits + assert(len == (len & 0x7f)); // At most 7 bits + return len | (copy_offset << 8) | (extra << 11); +} + +// Check that the decompression table is correct, and optionally print out +// the computed one. +TEST(Snappy, VerifyCharTable) { + using snappy::internal::LITERAL; + using snappy::internal::COPY_1_BYTE_OFFSET; + using snappy::internal::COPY_2_BYTE_OFFSET; + using snappy::internal::COPY_4_BYTE_OFFSET; + using snappy::internal::char_table; + + uint16 dst[256]; + + // Place invalid entries in all places to detect missing initialization + int assigned = 0; + for (int i = 0; i < 256; i++) { + dst[i] = 0xffff; + } + + // Small LITERAL entries. We store (len-1) in the top 6 bits. + for (unsigned int len = 1; len <= 60; len++) { + dst[LITERAL | ((len-1) << 2)] = MakeEntry(0, len, 0); + assigned++; + } + + // Large LITERAL entries. We use 60..63 in the high 6 bits to + // encode the number of bytes of length info that follow the opcode. + for (unsigned int extra_bytes = 1; extra_bytes <= 4; extra_bytes++) { + // We set the length field in the lookup table to 1 because extra + // bytes encode len-1. + dst[LITERAL | ((extra_bytes+59) << 2)] = MakeEntry(extra_bytes, 1, 0); + assigned++; + } + + // COPY_1_BYTE_OFFSET. + // + // The tag byte in the compressed data stores len-4 in 3 bits, and + // offset/256 in 5 bits. offset%256 is stored in the next byte. + // + // This format is used for length in range [4..11] and offset in + // range [0..2047] + for (unsigned int len = 4; len < 12; len++) { + for (unsigned int offset = 0; offset < 2048; offset += 256) { + dst[COPY_1_BYTE_OFFSET | ((len-4)<<2) | ((offset>>8)<<5)] = + MakeEntry(1, len, offset>>8); + assigned++; + } + } + + // COPY_2_BYTE_OFFSET. + // Tag contains len-1 in top 6 bits, and offset in next two bytes. + for (unsigned int len = 1; len <= 64; len++) { + dst[COPY_2_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(2, len, 0); + assigned++; + } + + // COPY_4_BYTE_OFFSET. + // Tag contents len-1 in top 6 bits, and offset in next four bytes. + for (unsigned int len = 1; len <= 64; len++) { + dst[COPY_4_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(4, len, 0); + assigned++; + } + + // Check that each entry was initialized exactly once. + EXPECT_EQ(256, assigned) << "Assigned only " << assigned << " of 256"; + for (int i = 0; i < 256; i++) { + EXPECT_NE(0xffff, dst[i]) << "Did not assign byte " << i; + } + + if (FLAGS_snappy_dump_decompression_table) { + printf("static const uint16 char_table[256] = {\n "); + for (int i = 0; i < 256; i++) { + printf("0x%04x%s", + dst[i], + ((i == 255) ? "\n" : (((i%8) == 7) ? ",\n " : ", "))); + } + printf("};\n"); + } + + // Check that computed table matched recorded table. + for (int i = 0; i < 256; i++) { + EXPECT_EQ(dst[i], char_table[i]) << "Mismatch in byte " << i; + } +} + +static void CompressFile(const char* fname) { + std::string fullinput; + CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults())); + + std::string compressed; + Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false); + + CHECK_OK(file::SetContents(std::string(fname).append(".comp"), compressed, + file::Defaults())); +} + +static void UncompressFile(const char* fname) { + std::string fullinput; + CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults())); + + size_t uncompLength; + CHECK(CheckUncompressedLength(fullinput, &uncompLength)); + + std::string uncompressed; + uncompressed.resize(uncompLength); + CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed)); + + CHECK_OK(file::SetContents(std::string(fname).append(".uncomp"), uncompressed, + file::Defaults())); +} + +static void MeasureFile(const char* fname) { + std::string fullinput; + CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults())); + printf("%-40s :\n", fname); + + int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len; + int end_len = fullinput.size(); + if (FLAGS_end_len >= 0) { + end_len = std::min(fullinput.size(), FLAGS_end_len); + } + for (int len = start_len; len <= end_len; len++) { + const char* const input = fullinput.data(); + int repeats = (FLAGS_bytes + len) / (len + 1); + if (FLAGS_zlib) Measure(input, len, ZLIB, repeats, 1024<<10); + if (FLAGS_lzo) Measure(input, len, LZO, repeats, 1024<<10); + if (FLAGS_snappy) Measure(input, len, SNAPPY, repeats, 4096<<10); + + // For block-size based measurements + if (0 && FLAGS_snappy) { + Measure(input, len, SNAPPY, repeats, 8<<10); + Measure(input, len, SNAPPY, repeats, 16<<10); + Measure(input, len, SNAPPY, repeats, 32<<10); + Measure(input, len, SNAPPY, repeats, 64<<10); + Measure(input, len, SNAPPY, repeats, 256<<10); + Measure(input, len, SNAPPY, repeats, 1024<<10); + } + } +} + +static struct { + const char* label; + const char* filename; + size_t size_limit; +} files[] = { + { "html", "html", 0 }, + { "urls", "urls.10K", 0 }, + { "jpg", "fireworks.jpeg", 0 }, + { "jpg_200", "fireworks.jpeg", 200 }, + { "pdf", "paper-100k.pdf", 0 }, + { "html4", "html_x_4", 0 }, + { "txt1", "alice29.txt", 0 }, + { "txt2", "asyoulik.txt", 0 }, + { "txt3", "lcet10.txt", 0 }, + { "txt4", "plrabn12.txt", 0 }, + { "pb", "geo.protodata", 0 }, + { "gaviota", "kppkn.gtb", 0 }, +}; + +static void BM_UFlat(int iters, int arg) { + StopBenchmarkTiming(); + + // Pick file to process based on "arg" + CHECK_GE(arg, 0); + CHECK_LT(arg, ARRAYSIZE(files)); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); + + std::string zcontents; + snappy::Compress(contents.data(), contents.size(), &zcontents); + char* dst = new char[contents.size()]; + + SetBenchmarkBytesProcessed(static_cast(iters) * + static_cast(contents.size())); + SetBenchmarkLabel(files[arg].label); + StartBenchmarkTiming(); + while (iters-- > 0) { + CHECK(snappy::RawUncompress(zcontents.data(), zcontents.size(), dst)); + } + StopBenchmarkTiming(); + + delete[] dst; +} +BENCHMARK(BM_UFlat)->DenseRange(0, ARRAYSIZE(files) - 1); + +static void BM_UValidate(int iters, int arg) { + StopBenchmarkTiming(); + + // Pick file to process based on "arg" + CHECK_GE(arg, 0); + CHECK_LT(arg, ARRAYSIZE(files)); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); + + std::string zcontents; + snappy::Compress(contents.data(), contents.size(), &zcontents); + + SetBenchmarkBytesProcessed(static_cast(iters) * + static_cast(contents.size())); + SetBenchmarkLabel(files[arg].label); + StartBenchmarkTiming(); + while (iters-- > 0) { + CHECK(snappy::IsValidCompressedBuffer(zcontents.data(), zcontents.size())); + } + StopBenchmarkTiming(); +} +BENCHMARK(BM_UValidate)->DenseRange(0, 4); + +static void BM_UIOVec(int iters, int arg) { + StopBenchmarkTiming(); + + // Pick file to process based on "arg" + CHECK_GE(arg, 0); + CHECK_LT(arg, ARRAYSIZE(files)); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); + + std::string zcontents; + snappy::Compress(contents.data(), contents.size(), &zcontents); + + // Uncompress into an iovec containing ten entries. + const int kNumEntries = 10; + struct iovec iov[kNumEntries]; + char *dst = new char[contents.size()]; + int used_so_far = 0; + for (int i = 0; i < kNumEntries; ++i) { + iov[i].iov_base = dst + used_so_far; + if (used_so_far == contents.size()) { + iov[i].iov_len = 0; + continue; + } + + if (i == kNumEntries - 1) { + iov[i].iov_len = contents.size() - used_so_far; + } else { + iov[i].iov_len = contents.size() / kNumEntries; + } + used_so_far += iov[i].iov_len; + } + + SetBenchmarkBytesProcessed(static_cast(iters) * + static_cast(contents.size())); + SetBenchmarkLabel(files[arg].label); + StartBenchmarkTiming(); + while (iters-- > 0) { + CHECK(snappy::RawUncompressToIOVec(zcontents.data(), zcontents.size(), iov, + kNumEntries)); + } + StopBenchmarkTiming(); + + delete[] dst; +} +BENCHMARK(BM_UIOVec)->DenseRange(0, 4); + +static void BM_UFlatSink(int iters, int arg) { + StopBenchmarkTiming(); + + // Pick file to process based on "arg" + CHECK_GE(arg, 0); + CHECK_LT(arg, ARRAYSIZE(files)); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); + + std::string zcontents; + snappy::Compress(contents.data(), contents.size(), &zcontents); + char* dst = new char[contents.size()]; + + SetBenchmarkBytesProcessed(static_cast(iters) * + static_cast(contents.size())); + SetBenchmarkLabel(files[arg].label); + StartBenchmarkTiming(); + while (iters-- > 0) { + snappy::ByteArraySource source(zcontents.data(), zcontents.size()); + snappy::UncheckedByteArraySink sink(dst); + CHECK(snappy::Uncompress(&source, &sink)); + } + StopBenchmarkTiming(); + + std::string s(dst, contents.size()); + CHECK_EQ(contents, s); + + delete[] dst; +} + +BENCHMARK(BM_UFlatSink)->DenseRange(0, ARRAYSIZE(files) - 1); + +static void BM_ZFlat(int iters, int arg) { + StopBenchmarkTiming(); + + // Pick file to process based on "arg" + CHECK_GE(arg, 0); + CHECK_LT(arg, ARRAYSIZE(files)); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); + + char* dst = new char[snappy::MaxCompressedLength(contents.size())]; + + SetBenchmarkBytesProcessed(static_cast(iters) * + static_cast(contents.size())); + StartBenchmarkTiming(); + + size_t zsize = 0; + while (iters-- > 0) { + snappy::RawCompress(contents.data(), contents.size(), dst, &zsize); + } + StopBenchmarkTiming(); + const double compression_ratio = + static_cast(zsize) / std::max(1, contents.size()); + SetBenchmarkLabel(StrFormat("%s (%.2f %%)", files[arg].label, + 100.0 * compression_ratio)); + VLOG(0) << StrFormat("compression for %s: %zd -> %zd bytes", + files[arg].label, static_cast(contents.size()), + static_cast(zsize)); + delete[] dst; +} +BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1); + +static void BM_ZFlatAll(int iters, int arg) { + StopBenchmarkTiming(); + + CHECK_EQ(arg, 0); + const int num_files = ARRAYSIZE(files); + + std::vector contents(num_files); + std::vector dst(num_files); + + int64 total_contents_size = 0; + for (int i = 0; i < num_files; ++i) { + contents[i] = ReadTestDataFile(files[i].filename, files[i].size_limit); + dst[i] = new char[snappy::MaxCompressedLength(contents[i].size())]; + total_contents_size += contents[i].size(); + } + + SetBenchmarkBytesProcessed(static_cast(iters) * total_contents_size); + StartBenchmarkTiming(); + + size_t zsize = 0; + while (iters-- > 0) { + for (int i = 0; i < num_files; ++i) { + snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i], + &zsize); + } + } + StopBenchmarkTiming(); + + for (int i = 0; i < num_files; ++i) { + delete[] dst[i]; + } + SetBenchmarkLabel(StrFormat("%d files", num_files)); +} +BENCHMARK(BM_ZFlatAll)->DenseRange(0, 0); + +static void BM_ZFlatIncreasingTableSize(int iters, int arg) { + StopBenchmarkTiming(); + + CHECK_EQ(arg, 0); + CHECK_GT(ARRAYSIZE(files), 0); + const std::string base_content = + ReadTestDataFile(files[0].filename, files[0].size_limit); + + std::vector contents; + std::vector dst; + int64 total_contents_size = 0; + for (int table_bits = kMinHashTableBits; table_bits <= kMaxHashTableBits; + ++table_bits) { + std::string content = base_content; + content.resize(1 << table_bits); + dst.push_back(new char[snappy::MaxCompressedLength(content.size())]); + total_contents_size += content.size(); + contents.push_back(std::move(content)); + } + + size_t zsize = 0; + SetBenchmarkBytesProcessed(static_cast(iters) * total_contents_size); + StartBenchmarkTiming(); + while (iters-- > 0) { + for (int i = 0; i < contents.size(); ++i) { + snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i], + &zsize); + } + } + StopBenchmarkTiming(); + + for (int i = 0; i < dst.size(); ++i) { + delete[] dst[i]; + } + SetBenchmarkLabel(StrFormat("%zd tables", contents.size())); +} +BENCHMARK(BM_ZFlatIncreasingTableSize)->DenseRange(0, 0); + +} // namespace snappy + +int main(int argc, char** argv) { + InitGoogle(argv[0], &argc, &argv, true); + RunSpecifiedBenchmarks(); + + if (argc >= 2) { + for (int arg = 1; arg < argc; arg++) { + if (FLAGS_write_compressed) { + snappy::CompressFile(argv[arg]); + } else if (FLAGS_write_uncompressed) { + snappy::UncompressFile(argv[arg]); + } else { + snappy::MeasureFile(argv[arg]); + } + } + return 0; + } + + return RUN_ALL_TESTS(); +} diff --git a/target/snappy-1.1.10/testdata/alice29.txt b/target/snappy-1.1.10/testdata/alice29.txt new file mode 100644 index 00000000..70336552 --- /dev/null +++ b/target/snappy-1.1.10/testdata/alice29.txt @@ -0,0 +1,3609 @@ + + + + + ALICE'S ADVENTURES IN WONDERLAND + + Lewis Carroll + + THE MILLENNIUM FULCRUM EDITION 2.9 + + + + + CHAPTER I + + Down the Rabbit-Hole + + + Alice was beginning to get very tired of sitting by her sister +on the bank, and of having nothing to do: once or twice she had +peeped into the book her sister was reading, but it had no +pictures or conversations in it, `and what is the use of a book,' +thought Alice `without pictures or conversation?' + + So she was considering in her own mind (as well as she could, +for the hot day made her feel very sleepy and stupid), whether +the pleasure of making a daisy-chain would be worth the trouble +of getting up and picking the daisies, when suddenly a White +Rabbit with pink eyes ran close by her. + + There was nothing so VERY remarkable in that; nor did Alice +think it so VERY much out of the way to hear the Rabbit say to +itself, `Oh dear! Oh dear! I shall be late!' (when she thought +it over afterwards, it occurred to her that she ought to have +wondered at this, but at the time it all seemed quite natural); +but when the Rabbit actually TOOK A WATCH OUT OF ITS WAISTCOAT- +POCKET, and looked at it, and then hurried on, Alice started to +her feet, for it flashed across her mind that she had never +before seen a rabbit with either a waistcoat-pocket, or a watch to +take out of it, and burning with curiosity, she ran across the +field after it, and fortunately was just in time to see it pop +down a large rabbit-hole under the hedge. + + In another moment down went Alice after it, never once +considering how in the world she was to get out again. + + The rabbit-hole went straight on like a tunnel for some way, +and then dipped suddenly down, so suddenly that Alice had not a +moment to think about stopping herself before she found herself +falling down a very deep well. + + Either the well was very deep, or she fell very slowly, for she +had plenty of time as she went down to look about her and to +wonder what was going to happen next. First, she tried to look +down and make out what she was coming to, but it was too dark to +see anything; then she looked at the sides of the well, and +noticed that they were filled with cupboards and book-shelves; +here and there she saw maps and pictures hung upon pegs. She +took down a jar from one of the shelves as she passed; it was +labelled `ORANGE MARMALADE', but to her great disappointment it +was empty: she did not like to drop the jar for fear of killing +somebody, so managed to put it into one of the cupboards as she +fell past it. + + `Well!' thought Alice to herself, `after such a fall as this, I +shall think nothing of tumbling down stairs! How brave they'll +all think me at home! Why, I wouldn't say anything about it, +even if I fell off the top of the house!' (Which was very likely +true.) + + Down, down, down. Would the fall NEVER come to an end! `I +wonder how many miles I've fallen by this time?' she said aloud. +`I must be getting somewhere near the centre of the earth. Let +me see: that would be four thousand miles down, I think--' (for, +you see, Alice had learnt several things of this sort in her +lessons in the schoolroom, and though this was not a VERY good +opportunity for showing off her knowledge, as there was no one to +listen to her, still it was good practice to say it over) `--yes, +that's about the right distance--but then I wonder what Latitude +or Longitude I've got to?' (Alice had no idea what Latitude was, +or Longitude either, but thought they were nice grand words to +say.) + + Presently she began again. `I wonder if I shall fall right +THROUGH the earth! How funny it'll seem to come out among the +people that walk with their heads downward! The Antipathies, I +think--' (she was rather glad there WAS no one listening, this +time, as it didn't sound at all the right word) `--but I shall +have to ask them what the name of the country is, you know. +Please, Ma'am, is this New Zealand or Australia?' (and she tried +to curtsey as she spoke--fancy CURTSEYING as you're falling +through the air! Do you think you could manage it?) `And what +an ignorant little girl she'll think me for asking! No, it'll +never do to ask: perhaps I shall see it written up somewhere.' + + Down, down, down. There was nothing else to do, so Alice soon +began talking again. `Dinah'll miss me very much to-night, I +should think!' (Dinah was the cat.) `I hope they'll remember +her saucer of milk at tea-time. Dinah my dear! I wish you were +down here with me! There are no mice in the air, I'm afraid, but +you might catch a bat, and that's very like a mouse, you know. +But do cats eat bats, I wonder?' And here Alice began to get +rather sleepy, and went on saying to herself, in a dreamy sort of +way, `Do cats eat bats? Do cats eat bats?' and sometimes, `Do +bats eat cats?' for, you see, as she couldn't answer either +question, it didn't much matter which way she put it. She felt +that she was dozing off, and had just begun to dream that she +was walking hand in hand with Dinah, and saying to her very +earnestly, `Now, Dinah, tell me the truth: did you ever eat a +bat?' when suddenly, thump! thump! down she came upon a heap of +sticks and dry leaves, and the fall was over. + + Alice was not a bit hurt, and she jumped up on to her feet in a +moment: she looked up, but it was all dark overhead; before her +was another long passage, and the White Rabbit was still in +sight, hurrying down it. There was not a moment to be lost: +away went Alice like the wind, and was just in time to hear it +say, as it turned a corner, `Oh my ears and whiskers, how late +it's getting!' She was close behind it when she turned the +corner, but the Rabbit was no longer to be seen: she found +herself in a long, low hall, which was lit up by a row of lamps +hanging from the roof. + + There were doors all round the hall, but they were all locked; +and when Alice had been all the way down one side and up the +other, trying every door, she walked sadly down the middle, +wondering how she was ever to get out again. + + Suddenly she came upon a little three-legged table, all made of +solid glass; there was nothing on it except a tiny golden key, +and Alice's first thought was that it might belong to one of the +doors of the hall; but, alas! either the locks were too large, or +the key was too small, but at any rate it would not open any of +them. However, on the second time round, she came upon a low +curtain she had not noticed before, and behind it was a little +door about fifteen inches high: she tried the little golden key +in the lock, and to her great delight it fitted! + + Alice opened the door and found that it led into a small +passage, not much larger than a rat-hole: she knelt down and +looked along the passage into the loveliest garden you ever saw. +How she longed to get out of that dark hall, and wander about +among those beds of bright flowers and those cool fountains, but +she could not even get her head though the doorway; `and even if +my head would go through,' thought poor Alice, `it would be of +very little use without my shoulders. Oh, how I wish +I could shut up like a telescope! I think I could, if I only +know how to begin.' For, you see, so many out-of-the-way things +had happened lately, that Alice had begun to think that very few +things indeed were really impossible. + + There seemed to be no use in waiting by the little door, so she +went back to the table, half hoping she might find another key on +it, or at any rate a book of rules for shutting people up like +telescopes: this time she found a little bottle on it, (`which +certainly was not here before,' said Alice,) and round the neck +of the bottle was a paper label, with the words `DRINK ME' +beautifully printed on it in large letters. + + It was all very well to say `Drink me,' but the wise little +Alice was not going to do THAT in a hurry. `No, I'll look +first,' she said, `and see whether it's marked "poison" or not'; +for she had read several nice little histories about children who +had got burnt, and eaten up by wild beasts and other unpleasant +things, all because they WOULD not remember the simple rules +their friends had taught them: such as, that a red-hot poker +will burn you if you hold it too long; and that if you cut your +finger VERY deeply with a knife, it usually bleeds; and she had +never forgotten that, if you drink much from a bottle marked +`poison,' it is almost certain to disagree with you, sooner or +later. + + However, this bottle was NOT marked `poison,' so Alice ventured +to taste it, and finding it very nice, (it had, in fact, a sort +of mixed flavour of cherry-tart, custard, pine-apple, roast +turkey, toffee, and hot buttered toast,) she very soon finished +it off. + + * * * * * * * + + * * * * * * + + * * * * * * * + + `What a curious feeling!' said Alice; `I must be shutting up +like a telescope.' + + And so it was indeed: she was now only ten inches high, and +her face brightened up at the thought that she was now the right +size for going though the little door into that lovely garden. +First, however, she waited for a few minutes to see if she was +going to shrink any further: she felt a little nervous about +this; `for it might end, you know,' said Alice to herself, `in my +going out altogether, like a candle. I wonder what I should be +like then?' And she tried to fancy what the flame of a candle is +like after the candle is blown out, for she could not remember +ever having seen such a thing. + + After a while, finding that nothing more happened, she decided +on going into the garden at once; but, alas for poor Alice! when +she got to the door, she found he had forgotten the little golden +key, and when she went back to the table for it, she found she +could not possibly reach it: she could see it quite plainly +through the glass, and she tried her best to climb up one of the +legs of the table, but it was too slippery; and when she had +tired herself out with trying, the poor little thing sat down and +cried. + + `Come, there's no use in crying like that!' said Alice to +herself, rather sharply; `I advise you to leave off this minute!' +She generally gave herself very good advice, (though she very +seldom followed it), and sometimes she scolded herself so +severely as to bring tears into her eyes; and once she remembered +trying to box her own ears for having cheated herself in a game +of croquet she was playing against herself, for this curious +child was very fond of pretending to be two people. `But it's no +use now,' thought poor Alice, `to pretend to be two people! Why, +there's hardly enough of me left to make ONE respectable +person!' + + Soon her eye fell on a little glass box that was lying under +the table: she opened it, and found in it a very small cake, on +which the words `EAT ME' were beautifully marked in currants. +`Well, I'll eat it,' said Alice, `and if it makes me grow larger, +I can reach the key; and if it makes me grow smaller, I can creep +under the door; so either way I'll get into the garden, and I +don't care which happens!' + + She ate a little bit, and said anxiously to herself, `Which +way? Which way?', holding her hand on the top of her head to +feel which way it was growing, and she was quite surprised to +find that she remained the same size: to be sure, this generally +happens when one eats cake, but Alice had got so much into the +way of expecting nothing but out-of-the-way things to happen, +that it seemed quite dull and stupid for life to go on in the +common way. + + So she set to work, and very soon finished off the cake. + + * * * * * * * + + * * * * * * + + * * * * * * * + + + + + CHAPTER II + + The Pool of Tears + + + `Curiouser and curiouser!' cried Alice (she was so much +surprised, that for the moment she quite forgot how to speak good +English); `now I'm opening out like the largest telescope that +ever was! Good-bye, feet!' (for when she looked down at her +feet, they seemed to be almost out of sight, they were getting so +far off). `Oh, my poor little feet, I wonder who will put on +your shoes and stockings for you now, dears? I'm sure _I_ shan't +be able! I shall be a great deal too far off to trouble myself +about you: you must manage the best way you can; --but I must be +kind to them,' thought Alice, `or perhaps they won't walk the +way I want to go! Let me see: I'll give them a new pair of +boots every Christmas.' + + And she went on planning to herself how she would manage it. +`They must go by the carrier,' she thought; `and how funny it'll +seem, sending presents to one's own feet! And how odd the +directions will look! + + ALICE'S RIGHT FOOT, ESQ. + HEARTHRUG, + NEAR THE FENDER, + (WITH ALICE'S LOVE). + +Oh dear, what nonsense I'm talking!' + + Just then her head struck against the roof of the hall: in +fact she was now more than nine feet high, and she at once took +up the little golden key and hurried off to the garden door. + + Poor Alice! It was as much as she could do, lying down on one +side, to look through into the garden with one eye; but to get +through was more hopeless than ever: she sat down and began to +cry again. + + `You ought to be ashamed of yourself,' said Alice, `a great +girl like you,' (she might well say this), `to go on crying in +this way! Stop this moment, I tell you!' But she went on all +the same, shedding gallons of tears, until there was a large pool +all round her, about four inches deep and reaching half down the +hall. + + After a time she heard a little pattering of feet in the +distance, and she hastily dried her eyes to see what was coming. +It was the White Rabbit returning, splendidly dressed, with a +pair of white kid gloves in one hand and a large fan in the +other: he came trotting along in a great hurry, muttering to +himself as he came, `Oh! the Duchess, the Duchess! Oh! won't she +be savage if I've kept her waiting!' Alice felt so desperate +that she was ready to ask help of any one; so, when the Rabbit +came near her, she began, in a low, timid voice, `If you please, +sir--' The Rabbit started violently, dropped the white kid +gloves and the fan, and skurried away into the darkness as hard +as he could go. + + Alice took up the fan and gloves, and, as the hall was very +hot, she kept fanning herself all the time she went on talking: +`Dear, dear! How queer everything is to-day! And yesterday +things went on just as usual. I wonder if I've been changed in +the night? Let me think: was I the same when I got up this +morning? I almost think I can remember feeling a little +different. But if I'm not the same, the next question is, Who in +the world am I? Ah, THAT'S the great puzzle!' And she began +thinking over all the children she knew that were of the same age +as herself, to see if she could have been changed for any of +them. + + `I'm sure I'm not Ada,' she said, `for her hair goes in such +long ringlets, and mine doesn't go in ringlets at all; and I'm +sure I can't be Mabel, for I know all sorts of things, and she, +oh! she knows such a very little! Besides, SHE'S she, and I'm I, +and--oh dear, how puzzling it all is! I'll try if I know all the +things I used to know. Let me see: four times five is twelve, +and four times six is thirteen, and four times seven is--oh dear! +I shall never get to twenty at that rate! However, the +Multiplication Table doesn't signify: let's try Geography. +London is the capital of Paris, and Paris is the capital of Rome, +and Rome--no, THAT'S all wrong, I'm certain! I must have been +changed for Mabel! I'll try and say "How doth the little--"' +and she crossed her hands on her lap as if she were saying lessons, +and began to repeat it, but her voice sounded hoarse and +strange, and the words did not come the same as they used to do:-- + + `How doth the little crocodile + Improve his shining tail, + And pour the waters of the Nile + On every golden scale! + + `How cheerfully he seems to grin, + How neatly spread his claws, + And welcome little fishes in + With gently smiling jaws!' + + `I'm sure those are not the right words,' said poor Alice, and +her eyes filled with tears again as she went on, `I must be Mabel +after all, and I shall have to go and live in that poky little +house, and have next to no toys to play with, and oh! ever so +many lessons to learn! No, I've made up my mind about it; if I'm +Mabel, I'll stay down here! It'll be no use their putting their +heads down and saying "Come up again, dear!" I shall only look +up and say "Who am I then? Tell me that first, and then, if I +like being that person, I'll come up: if not, I'll stay down +here till I'm somebody else"--but, oh dear!' cried Alice, with a +sudden burst of tears, `I do wish they WOULD put their heads +down! I am so VERY tired of being all alone here!' + + As she said this she looked down at her hands, and was +surprised to see that she had put on one of the Rabbit's little +white kid gloves while she was talking. `How CAN I have done +that?' she thought. `I must be growing small again.' She got up +and went to the table to measure herself by it, and found that, +as nearly as she could guess, she was now about two feet high, +and was going on shrinking rapidly: she soon found out that the +cause of this was the fan she was holding, and she dropped it +hastily, just in time to avoid shrinking away altogether. + +`That WAS a narrow escape!' said Alice, a good deal frightened at +the sudden change, but very glad to find herself still in +existence; `and now for the garden!' and she ran with all speed +back to the little door: but, alas! the little door was shut +again, and the little golden key was lying on the glass table as +before, `and things are worse than ever,' thought the poor child, +`for I never was so small as this before, never! And I declare +it's too bad, that it is!' + + As she said these words her foot slipped, and in another +moment, splash! she was up to her chin in salt water. He first +idea was that she had somehow fallen into the sea, `and in that +case I can go back by railway,' she said to herself. (Alice had +been to the seaside once in her life, and had come to the general +conclusion, that wherever you go to on the English coast you find +a number of bathing machines in the sea, some children digging in +the sand with wooden spades, then a row of lodging houses, and +behind them a railway station.) However, she soon made out that +she was in the pool of tears which she had wept when she was nine +feet high. + + `I wish I hadn't cried so much!' said Alice, as she swam about, +trying to find her way out. `I shall be punished for it now, I +suppose, by being drowned in my own tears! That WILL be a queer +thing, to be sure! However, everything is queer to-day.' + + Just then she heard something splashing about in the pool a +little way off, and she swam nearer to make out what it was: at +first she thought it must be a walrus or hippopotamus, but then +she remembered how small she was now, and she soon made out that +it was only a mouse that had slipped in like herself. + + `Would it be of any use, now,' thought Alice, `to speak to this +mouse? Everything is so out-of-the-way down here, that I should +think very likely it can talk: at any rate, there's no harm in +trying.' So she began: `O Mouse, do you know the way out of +this pool? I am very tired of swimming about here, O Mouse!' +(Alice thought this must be the right way of speaking to a mouse: +she had never done such a thing before, but she remembered having +seen in her brother's Latin Grammar, `A mouse--of a mouse--to a +mouse--a mouse--O mouse!' The Mouse looked at her rather +inquisitively, and seemed to her to wink with one of its little +eyes, but it said nothing. + + `Perhaps it doesn't understand English,' thought Alice; `I +daresay it's a French mouse, come over with William the +Conqueror.' (For, with all her knowledge of history, Alice had +no very clear notion how long ago anything had happened.) So she +began again: `Ou est ma chatte?' which was the first sentence in +her French lesson-book. The Mouse gave a sudden leap out of the +water, and seemed to quiver all over with fright. `Oh, I beg +your pardon!' cried Alice hastily, afraid that she had hurt the +poor animal's feelings. `I quite forgot you didn't like cats.' + + `Not like cats!' cried the Mouse, in a shrill, passionate +voice. `Would YOU like cats if you were me?' + + `Well, perhaps not,' said Alice in a soothing tone: `don't be +angry about it. And yet I wish I could show you our cat Dinah: +I think you'd take a fancy to cats if you could only see her. +She is such a dear quiet thing,' Alice went on, half to herself, +as she swam lazily about in the pool, `and she sits purring so +nicely by the fire, licking her paws and washing her face--and +she is such a nice soft thing to nurse--and she's such a capital +one for catching mice--oh, I beg your pardon!' cried Alice again, +for this time the Mouse was bristling all over, and she felt +certain it must be really offended. `We won't talk about her any +more if you'd rather not.' + + `We indeed!' cried the Mouse, who was trembling down to the end +of his tail. `As if I would talk on such a subject! Our family +always HATED cats: nasty, low, vulgar things! Don't let me hear +the name again!' + + `I won't indeed!' said Alice, in a great hurry to change the +subject of conversation. `Are you--are you fond--of--of dogs?' +The Mouse did not answer, so Alice went on eagerly: `There is +such a nice little dog near our house I should like to show you! +A little bright-eyed terrier, you know, with oh, such long curly +brown hair! And it'll fetch things when you throw them, and +it'll sit up and beg for its dinner, and all sorts of things--I +can't remember half of them--and it belongs to a farmer, you +know, and he says it's so useful, it's worth a hundred pounds! +He says it kills all the rats and--oh dear!' cried Alice in a +sorrowful tone, `I'm afraid I've offended it again!' For the +Mouse was swimming away from her as hard as it could go, and +making quite a commotion in the pool as it went. + + So she called softly after it, `Mouse dear! Do come back +again, and we won't talk about cats or dogs either, if you don't +like them!' When the Mouse heard this, it turned round and swam +slowly back to her: its face was quite pale (with passion, Alice +thought), and it said in a low trembling voice, `Let us get to +the shore, and then I'll tell you my history, and you'll +understand why it is I hate cats and dogs.' + + It was high time to go, for the pool was getting quite crowded +with the birds and animals that had fallen into it: there were a +Duck and a Dodo, a Lory and an Eaglet, and several other curious +creatures. Alice led the way, and the whole party swam to the +shore. + + + + CHAPTER III + + A Caucus-Race and a Long Tale + + + They were indeed a queer-looking party that assembled on the +bank--the birds with draggled feathers, the animals with their +fur clinging close to them, and all dripping wet, cross, and +uncomfortable. + + The first question of course was, how to get dry again: they +had a consultation about this, and after a few minutes it seemed +quite natural to Alice to find herself talking familiarly with +them, as if she had known them all her life. Indeed, she had +quite a long argument with the Lory, who at last turned sulky, +and would only say, `I am older than you, and must know better'; +and this Alice would not allow without knowing how old it was, +and, as the Lory positively refused to tell its age, there was no +more to be said. + + At last the Mouse, who seemed to be a person of authority among +them, called out, `Sit down, all of you, and listen to me! I'LL +soon make you dry enough!' They all sat down at once, in a large +ring, with the Mouse in the middle. Alice kept her eyes +anxiously fixed on it, for she felt sure she would catch a bad +cold if she did not get dry very soon. + + `Ahem!' said the Mouse with an important air, `are you all ready? +This is the driest thing I know. Silence all round, if you please! +"William the Conqueror, whose cause was favoured by the pope, was +soon submitted to by the English, who wanted leaders, and had been +of late much accustomed to usurpation and conquest. Edwin and +Morcar, the earls of Mercia and Northumbria--"' + + `Ugh!' said the Lory, with a shiver. + + `I beg your pardon!' said the Mouse, frowning, but very +politely: `Did you speak?' + + `Not I!' said the Lory hastily. + + `I thought you did,' said the Mouse. `--I proceed. "Edwin and +Morcar, the earls of Mercia and Northumbria, declared for him: +and even Stigand, the patriotic archbishop of Canterbury, found +it advisable--"' + + `Found WHAT?' said the Duck. + + `Found IT,' the Mouse replied rather crossly: `of course you +know what "it" means.' + + `I know what "it" means well enough, when I find a thing,' said +the Duck: `it's generally a frog or a worm. The question is, +what did the archbishop find?' + + The Mouse did not notice this question, but hurriedly went on, +`"--found it advisable to go with Edgar Atheling to meet William +and offer him the crown. William's conduct at first was +moderate. But the insolence of his Normans--" How are you +getting on now, my dear?' it continued, turning to Alice as it +spoke. + + `As wet as ever,' said Alice in a melancholy tone: `it doesn't +seem to dry me at all.' + + `In that case,' said the Dodo solemnly, rising to its feet, `I +move that the meeting adjourn, for the immediate adoption of more +energetic remedies--' + + `Speak English!' said the Eaglet. `I don't know the meaning of +half those long words, and, what's more, I don't believe you do +either!' And the Eaglet bent down its head to hide a smile: +some of the other birds tittered audibly. + + `What I was going to say,' said the Dodo in an offended tone, +`was, that the best thing to get us dry would be a Caucus-race.' + + `What IS a Caucus-race?' said Alice; not that she wanted much +to know, but the Dodo had paused as if it thought that SOMEBODY +ought to speak, and no one else seemed inclined to say anything. + + `Why,' said the Dodo, `the best way to explain it is to do it.' +(And, as you might like to try the thing yourself, some winter +day, I will tell you how the Dodo managed it.) + + First it marked out a race-course, in a sort of circle, (`the +exact shape doesn't matter,' it said,) and then all the party +were placed along the course, here and there. There was no `One, +two, three, and away,' but they began running when they liked, +and left off when they liked, so that it was not easy to know +when the race was over. However, when they had been running half +an hour or so, and were quite dry again, the Dodo suddenly called +out `The race is over!' and they all crowded round it, panting, +and asking, `But who has won?' + + This question the Dodo could not answer without a great deal of +thought, and it sat for a long time with one finger pressed upon +its forehead (the position in which you usually see Shakespeare, +in the pictures of him), while the rest waited in silence. At +last the Dodo said, `EVERYBODY has won, and all must have +prizes.' + + `But who is to give the prizes?' quite a chorus of voices +asked. + + `Why, SHE, of course,' said the Dodo, pointing to Alice with +one finger; and the whole party at once crowded round her, +calling out in a confused way, `Prizes! Prizes!' + + Alice had no idea what to do, and in despair she put her hand +in her pocket, and pulled out a box of comfits, (luckily the salt +water had not got into it), and handed them round as prizes. +There was exactly one a-piece all round. + + `But she must have a prize herself, you know,' said the Mouse. + + `Of course,' the Dodo replied very gravely. `What else have +you got in your pocket?' he went on, turning to Alice. + + `Only a thimble,' said Alice sadly. + + `Hand it over here,' said the Dodo. + + Then they all crowded round her once more, while the Dodo +solemnly presented the thimble, saying `We beg your acceptance of +this elegant thimble'; and, when it had finished this short +speech, they all cheered. + + Alice thought the whole thing very absurd, but they all looked +so grave that she did not dare to laugh; and, as she could not +think of anything to say, she simply bowed, and took the thimble, +looking as solemn as she could. + + The next thing was to eat the comfits: this caused some noise +and confusion, as the large birds complained that they could not +taste theirs, and the small ones choked and had to be patted on +the back. However, it was over at last, and they sat down again +in a ring, and begged the Mouse to tell them something more. + + `You promised to tell me your history, you know,' said Alice, +`and why it is you hate--C and D,' she added in a whisper, half +afraid that it would be offended again. + + `Mine is a long and a sad tale!' said the Mouse, turning to +Alice, and sighing. + + `It IS a long tail, certainly,' said Alice, looking down with +wonder at the Mouse's tail; `but why do you call it sad?' And +she kept on puzzling about it while the Mouse was speaking, so +that her idea of the tale was something like this:-- + + `Fury said to a + mouse, That he + met in the + house, + "Let us + both go to + law: I will + prosecute + YOU. --Come, + I'll take no + denial; We + must have a + trial: For + really this + morning I've + nothing + to do." + Said the + mouse to the + cur, "Such + a trial, + dear Sir, + With + no jury + or judge, + would be + wasting + our + breath." + "I'll be + judge, I'll + be jury," + Said + cunning + old Fury: + "I'll + try the + whole + cause, + and + condemn + you + to + death."' + + + `You are not attending!' said the Mouse to Alice severely. +`What are you thinking of?' + + `I beg your pardon,' said Alice very humbly: `you had got to +the fifth bend, I think?' + + `I had NOT!' cried the Mouse, sharply and very angrily. + + `A knot!' said Alice, always ready to make herself useful, and +looking anxiously about her. `Oh, do let me help to undo it!' + + `I shall do nothing of the sort,' said the Mouse, getting up +and walking away. `You insult me by talking such nonsense!' + + `I didn't mean it!' pleaded poor Alice. `But you're so easily +offended, you know!' + + The Mouse only growled in reply. + + `Please come back and finish your story!' Alice called after +it; and the others all joined in chorus, `Yes, please do!' but +the Mouse only shook its head impatiently, and walked a little +quicker. + + `What a pity it wouldn't stay!' sighed the Lory, as soon as it +was quite out of sight; and an old Crab took the opportunity of +saying to her daughter `Ah, my dear! Let this be a lesson to you +never to lose YOUR temper!' `Hold your tongue, Ma!' said the +young Crab, a little snappishly. `You're enough to try the +patience of an oyster!' + + `I wish I had our Dinah here, I know I do!' said Alice aloud, +addressing nobody in particular. `She'd soon fetch it back!' + + `And who is Dinah, if I might venture to ask the question?' +said the Lory. + + Alice replied eagerly, for she was always ready to talk about +her pet: `Dinah's our cat. And she's such a capital one for +catching mice you can't think! And oh, I wish you could see her +after the birds! Why, she'll eat a little bird as soon as look +at it!' + + This speech caused a remarkable sensation among the party. +Some of the birds hurried off at once: one the old Magpie began +wrapping itself up very carefully, remarking, `I really must be +getting home; the night-air doesn't suit my throat!' and a Canary +called out in a trembling voice to its children, `Come away, my +dears! It's high time you were all in bed!' On various pretexts +they all moved off, and Alice was soon left alone. + + `I wish I hadn't mentioned Dinah!' she said to herself in a +melancholy tone. `Nobody seems to like her, down here, and I'm +sure she's the best cat in the world! Oh, my dear Dinah! I +wonder if I shall ever see you any more!' And here poor Alice +began to cry again, for she felt very lonely and low-spirited. +In a little while, however, she again heard a little pattering of +footsteps in the distance, and she looked up eagerly, half hoping +that the Mouse had changed his mind, and was coming back to +finish his story. + + + + CHAPTER IV + + The Rabbit Sends in a Little Bill + + + It was the White Rabbit, trotting slowly back again, and +looking anxiously about as it went, as if it had lost something; +and she heard it muttering to itself `The Duchess! The Duchess! +Oh my dear paws! Oh my fur and whiskers! She'll get me +executed, as sure as ferrets are ferrets! Where CAN I have +dropped them, I wonder?' Alice guessed in a moment that it was +looking for the fan and the pair of white kid gloves, and she +very good-naturedly began hunting about for them, but they were +nowhere to be seen--everything seemed to have changed since her +swim in the pool, and the great hall, with the glass table and +the little door, had vanished completely. + + Very soon the Rabbit noticed Alice, as she went hunting about, +and called out to her in an angry tone, `Why, Mary Ann, what ARE +you doing out here? Run home this moment, and fetch me a pair of +gloves and a fan! Quick, now!' And Alice was so much frightened +that she ran off at once in the direction it pointed to, without +trying to explain the mistake it had made. + + `He took me for his housemaid,' she said to herself as she ran. +`How surprised he'll be when he finds out who I am! But I'd +better take him his fan and gloves--that is, if I can find them.' +As she said this, she came upon a neat little house, on the door +of which was a bright brass plate with the name `W. RABBIT' +engraved upon it. She went in without knocking, and hurried +upstairs, in great fear lest she should meet the real Mary Ann, +and be turned out of the house before she had found the fan and +gloves. + + `How queer it seems,' Alice said to herself, `to be going +messages for a rabbit! I suppose Dinah'll be sending me on +messages next!' And she began fancying the sort of thing that +would happen: `"Miss Alice! Come here directly, and get ready +for your walk!" "Coming in a minute, nurse! But I've got to see +that the mouse doesn't get out." Only I don't think,' Alice went +on, `that they'd let Dinah stop in the house if it began ordering +people about like that!' + + By this time she had found her way into a tidy little room with +a table in the window, and on it (as she had hoped) a fan and two +or three pairs of tiny white kid gloves: she took up the fan and +a pair of the gloves, and was just going to leave the room, when +her eye fell upon a little bottle that stood near the looking- +glass. There was no label this time with the words `DRINK ME,' +but nevertheless she uncorked it and put it to her lips. `I know +SOMETHING interesting is sure to happen,' she said to herself, +`whenever I eat or drink anything; so I'll just see what this +bottle does. I do hope it'll make me grow large again, for +really I'm quite tired of being such a tiny little thing!' + + It did so indeed, and much sooner than she had expected: +before she had drunk half the bottle, she found her head pressing +against the ceiling, and had to stoop to save her neck from being +broken. She hastily put down the bottle, saying to herself +`That's quite enough--I hope I shan't grow any more--As it is, I +can't get out at the door--I do wish I hadn't drunk quite so +much!' + + Alas! it was too late to wish that! She went on growing, and +growing, and very soon had to kneel down on the floor: in +another minute there was not even room for this, and she tried +the effect of lying down with one elbow against the door, and the +other arm curled round her head. Still she went on growing, and, +as a last resource, she put one arm out of the window, and one +foot up the chimney, and said to herself `Now I can do no more, +whatever happens. What WILL become of me?' + + Luckily for Alice, the little magic bottle had now had its full +effect, and she grew no larger: still it was very uncomfortable, +and, as there seemed to be no sort of chance of her ever getting +out of the room again, no wonder she felt unhappy. + + `It was much pleasanter at home,' thought poor Alice, `when one +wasn't always growing larger and smaller, and being ordered about +by mice and rabbits. I almost wish I hadn't gone down that +rabbit-hole--and yet--and yet--it's rather curious, you know, +this sort of life! I do wonder what CAN have happened to me! +When I used to read fairy-tales, I fancied that kind of thing +never happened, and now here I am in the middle of one! There +ought to be a book written about me, that there ought! And when +I grow up, I'll write one--but I'm grown up now,' she added in a +sorrowful tone; `at least there's no room to grow up any more +HERE.' + + `But then,' thought Alice, `shall I NEVER get any older than I +am now? That'll be a comfort, one way--never to be an old woman- +-but then--always to have lessons to learn! Oh, I shouldn't like +THAT!' + + `Oh, you foolish Alice!' she answered herself. `How can you +learn lessons in here? Why, there's hardly room for YOU, and no +room at all for any lesson-books!' + + And so she went on, taking first one side and then the other, +and making quite a conversation of it altogether; but after a few +minutes she heard a voice outside, and stopped to listen. + + `Mary Ann! Mary Ann!' said the voice. `Fetch me my gloves +this moment!' Then came a little pattering of feet on the +stairs. Alice knew it was the Rabbit coming to look for her, and +she trembled till she shook the house, quite forgetting that she +was now about a thousand times as large as the Rabbit, and had no +reason to be afraid of it. + + Presently the Rabbit came up to the door, and tried to open it; +but, as the door opened inwards, and Alice's elbow was pressed +hard against it, that attempt proved a failure. Alice heard it +say to itself `Then I'll go round and get in at the window.' + + `THAT you won't' thought Alice, and, after waiting till she +fancied she heard the Rabbit just under the window, she suddenly +spread out her hand, and made a snatch in the air. She did not +get hold of anything, but she heard a little shriek and a fall, +and a crash of broken glass, from which she concluded that it was +just possible it had fallen into a cucumber-frame, or something +of the sort. + + Next came an angry voice--the Rabbit's--`Pat! Pat! Where are +you?' And then a voice she had never heard before, `Sure then +I'm here! Digging for apples, yer honour!' + + `Digging for apples, indeed!' said the Rabbit angrily. `Here! +Come and help me out of THIS!' (Sounds of more broken glass.) + + `Now tell me, Pat, what's that in the window?' + + `Sure, it's an arm, yer honour!' (He pronounced it `arrum.') + + `An arm, you goose! Who ever saw one that size? Why, it +fills the whole window!' + + `Sure, it does, yer honour: but it's an arm for all that.' + + `Well, it's got no business there, at any rate: go and take it +away!' + + There was a long silence after this, and Alice could only hear +whispers now and then; such as, `Sure, I don't like it, yer +honour, at all, at all!' `Do as I tell you, you coward!' and at +last she spread out her hand again, and made another snatch in +the air. This time there were TWO little shrieks, and more +sounds of broken glass. `What a number of cucumber-frames there +must be!' thought Alice. `I wonder what they'll do next! As for +pulling me out of the window, I only wish they COULD! I'm sure I +don't want to stay in here any longer!' + + She waited for some time without hearing anything more: at +last came a rumbling of little cartwheels, and the sound of a +good many voice all talking together: she made out the words: +`Where's the other ladder?--Why, I hadn't to bring but one; +Bill's got the other--Bill! fetch it here, lad!--Here, put 'em up +at this corner--No, tie 'em together first--they don't reach half +high enough yet--Oh! they'll do well enough; don't be particular- +-Here, Bill! catch hold of this rope--Will the roof bear?--Mind +that loose slate--Oh, it's coming down! Heads below!' (a loud +crash)--`Now, who did that?--It was Bill, I fancy--Who's to go +down the chimney?--Nay, I shan't! YOU do it!--That I won't, +then!--Bill's to go down--Here, Bill! the master says you're to +go down the chimney!' + + `Oh! So Bill's got to come down the chimney, has he?' said +Alice to herself. `Shy, they seem to put everything upon Bill! +I wouldn't be in Bill's place for a good deal: this fireplace is +narrow, to be sure; but I THINK I can kick a little!' + + She drew her foot as far down the chimney as she could, and +waited till she heard a little animal (she couldn't guess of what +sort it was) scratching and scrambling about in the chimney close +above her: then, saying to herself `This is Bill,' she gave one +sharp kick, and waited to see what would happen next. + + The first thing she heard was a general chorus of `There goes +Bill!' then the Rabbit's voice along--`Catch him, you by the +hedge!' then silence, and then another confusion of voices--`Hold +up his head--Brandy now--Don't choke him--How was it, old fellow? +What happened to you? Tell us all about it!' + + Last came a little feeble, squeaking voice, (`That's Bill,' +thought Alice,) `Well, I hardly know--No more, thank ye; I'm +better now--but I'm a deal too flustered to tell you--all I know +is, something comes at me like a Jack-in-the-box, and up I goes +like a sky-rocket!' + + `So you did, old fellow!' said the others. + + `We must burn the house down!' said the Rabbit's voice; and +Alice called out as loud as she could, `If you do. I'll set +Dinah at you!' + + There was a dead silence instantly, and Alice thought to +herself, `I wonder what they WILL do next! If they had any +sense, they'd take the roof off.' After a minute or two, they +began moving about again, and Alice heard the Rabbit say, `A +barrowful will do, to begin with.' + + `A barrowful of WHAT?' thought Alice; but she had not long to +doubt, for the next moment a shower of little pebbles came +rattling in at the window, and some of them hit her in the face. +`I'll put a stop to this,' she said to herself, and shouted out, +`You'd better not do that again!' which produced another dead +silence. + + Alice noticed with some surprise that the pebbles were all +turning into little cakes as they lay on the floor, and a bright +idea came into her head. `If I eat one of these cakes,' she +thought, `it's sure to make SOME change in my size; and as it +can't possibly make me larger, it must make me smaller, I +suppose.' + + So she swallowed one of the cakes, and was delighted to find +that she began shrinking directly. As soon as she was small +enough to get through the door, she ran out of the house, and +found quite a crowd of little animals and birds waiting outside. +The poor little Lizard, Bill, was in the middle, being held up by +two guinea-pigs, who were giving it something out of a bottle. +They all made a rush at Alice the moment she appeared; but she +ran off as hard as she could, and soon found herself safe in a +thick wood. + + `The first thing I've got to do,' said Alice to herself, as she +wandered about in the wood, `is to grow to my right size again; +and the second thing is to find my way into that lovely garden. +I think that will be the best plan.' + + It sounded an excellent plan, no doubt, and very neatly and +simply arranged; the only difficulty was, that she had not the +smallest idea how to set about it; and while she was peering +about anxiously among the trees, a little sharp bark just over +her head made her look up in a great hurry. + + An enormous puppy was looking down at her with large round +eyes, and feebly stretching out one paw, trying to touch her. +`Poor little thing!' said Alice, in a coaxing tone, and she tried +hard to whistle to it; but she was terribly frightened all the +time at the thought that it might be hungry, in which case it +would be very likely to eat her up in spite of all her coaxing. + + Hardly knowing what she did, she picked up a little bit of +stick, and held it out to the puppy; whereupon the puppy jumped +into the air off all its feet at once, with a yelp of delight, +and rushed at the stick, and made believe to worry it; then Alice +dodged behind a great thistle, to keep herself from being run +over; and the moment she appeared on the other side, the puppy +made another rush at the stick, and tumbled head over heels in +its hurry to get hold of it; then Alice, thinking it was very +like having a game of play with a cart-horse, and expecting every +moment to be trampled under its feet, ran round the thistle +again; then the puppy began a series of short charges at the +stick, running a very little way forwards each time and a long +way back, and barking hoarsely all the while, till at last it sat +down a good way off, panting, with its tongue hanging out of its +mouth, and its great eyes half shut. + + This seemed to Alice a good opportunity for making her escape; +so she set off at once, and ran till she was quite tired and out +of breath, and till the puppy's bark sounded quite faint in the +distance. + + `And yet what a dear little puppy it was!' said Alice, as she +leant against a buttercup to rest herself, and fanned herself +with one of the leaves: `I should have liked teaching it tricks +very much, if--if I'd only been the right size to do it! Oh +dear! I'd nearly forgotten that I've got to grow up again! Let +me see--how IS it to be managed? I suppose I ought to eat or +drink something or other; but the great question is, what?' + + The great question certainly was, what? Alice looked all round +her at the flowers and the blades of grass, but she did not see +anything that looked like the right thing to eat or drink under +the circumstances. There was a large mushroom growing near her, +about the same height as herself; and when she had looked under +it, and on both sides of it, and behind it, it occurred to her +that she might as well look and see what was on the top of it. + + She stretched herself up on tiptoe, and peeped over the edge of +the mushroom, and her eyes immediately met those of a large +caterpillar, that was sitting on the top with its arms folded, +quietly smoking a long hookah, and taking not the smallest notice +of her or of anything else. + + + + CHAPTER V + + Advice from a Caterpillar + + + The Caterpillar and Alice looked at each other for some time in +silence: at last the Caterpillar took the hookah out of its +mouth, and addressed her in a languid, sleepy voice. + + `Who are YOU?' said the Caterpillar. + + This was not an encouraging opening for a conversation. Alice +replied, rather shyly, `I--I hardly know, sir, just at present-- +at least I know who I WAS when I got up this morning, but I think +I must have been changed several times since then.' + + `What do you mean by that?' said the Caterpillar sternly. +`Explain yourself!' + + `I can't explain MYSELF, I'm afraid, sir' said Alice, `because +I'm not myself, you see.' + + `I don't see,' said the Caterpillar. + + `I'm afraid I can't put it more clearly,' Alice replied very +politely, `for I can't understand it myself to begin with; and +being so many different sizes in a day is very confusing.' + + `It isn't,' said the Caterpillar. + + `Well, perhaps you haven't found it so yet,' said Alice; `but +when you have to turn into a chrysalis--you will some day, you +know--and then after that into a butterfly, I should think you'll +feel it a little queer, won't you?' + + `Not a bit,' said the Caterpillar. + + `Well, perhaps your feelings may be different,' said Alice; +`all I know is, it would feel very queer to ME.' + + `You!' said the Caterpillar contemptuously. `Who are YOU?' + + Which brought them back again to the beginning of the +conversation. Alice felt a little irritated at the Caterpillar's +making such VERY short remarks, and she drew herself up and said, +very gravely, `I think, you ought to tell me who YOU are, first.' + + `Why?' said the Caterpillar. + + Here was another puzzling question; and as Alice could not +think of any good reason, and as the Caterpillar seemed to be in +a VERY unpleasant state of mind, she turned away. + + `Come back!' the Caterpillar called after her. `I've something +important to say!' + + This sounded promising, certainly: Alice turned and came back +again. + + `Keep your temper,' said the Caterpillar. + + `Is that all?' said Alice, swallowing down her anger as well as +she could. + + `No,' said the Caterpillar. + + Alice thought she might as well wait, as she had nothing else +to do, and perhaps after all it might tell her something worth +hearing. For some minutes it puffed away without speaking, but +at last it unfolded its arms, took the hookah out of its mouth +again, and said, `So you think you're changed, do you?' + + `I'm afraid I am, sir,' said Alice; `I can't remember things as +I used--and I don't keep the same size for ten minutes together!' + + `Can't remember WHAT things?' said the Caterpillar. + + `Well, I've tried to say "HOW DOTH THE LITTLE BUSY BEE," but it +all came different!' Alice replied in a very melancholy voice. + + `Repeat, "YOU ARE OLD, FATHER WILLIAM,"' said the Caterpillar. + + Alice folded her hands, and began:-- + + `You are old, Father William,' the young man said, + `And your hair has become very white; + And yet you incessantly stand on your head-- + Do you think, at your age, it is right?' + + `In my youth,' Father William replied to his son, + `I feared it might injure the brain; + But, now that I'm perfectly sure I have none, + Why, I do it again and again.' + + `You are old,' said the youth, `as I mentioned before, + And have grown most uncommonly fat; + Yet you turned a back-somersault in at the door-- + Pray, what is the reason of that?' + + `In my youth,' said the sage, as he shook his grey locks, + `I kept all my limbs very supple + By the use of this ointment--one shilling the box-- + Allow me to sell you a couple?' + + `You are old,' said the youth, `and your jaws are too weak + For anything tougher than suet; + Yet you finished the goose, with the bones and the beak-- + Pray how did you manage to do it?' + + `In my youth,' said his father, `I took to the law, + And argued each case with my wife; + And the muscular strength, which it gave to my jaw, + Has lasted the rest of my life.' + + `You are old,' said the youth, `one would hardly suppose + That your eye was as steady as ever; + Yet you balanced an eel on the end of your nose-- + What made you so awfully clever?' + + `I have answered three questions, and that is enough,' + Said his father; `don't give yourself airs! + Do you think I can listen all day to such stuff? + Be off, or I'll kick you down stairs!' + + + `That is not said right,' said the Caterpillar. + + `Not QUITE right, I'm afraid,' said Alice, timidly; `some of the +words have got altered.' + + `It is wrong from beginning to end,' said the Caterpillar +decidedly, and there was silence for some minutes. + + The Caterpillar was the first to speak. + + `What size do you want to be?' it asked. + + `Oh, I'm not particular as to size,' Alice hastily replied; +`only one doesn't like changing so often, you know.' + + `I DON'T know,' said the Caterpillar. + + Alice said nothing: she had never been so much contradicted in +her life before, and she felt that she was losing her temper. + + `Are you content now?' said the Caterpillar. + + `Well, I should like to be a LITTLE larger, sir, if you +wouldn't mind,' said Alice: `three inches is such a wretched +height to be.' + + `It is a very good height indeed!' said the Caterpillar +angrily, rearing itself upright as it spoke (it was exactly three +inches high). + + `But I'm not used to it!' pleaded poor Alice in a piteous tone. +And she thought of herself, `I wish the creatures wouldn't be so +easily offended!' + + `You'll get used to it in time,' said the Caterpillar; and it +put the hookah into its mouth and began smoking again. + + This time Alice waited patiently until it chose to speak again. +In a minute or two the Caterpillar took the hookah out of its +mouth and yawned once or twice, and shook itself. Then it got +down off the mushroom, and crawled away in the grass, merely +remarking as it went, `One side will make you grow taller, and +the other side will make you grow shorter.' + + `One side of WHAT? The other side of WHAT?' thought Alice to +herself. + + `Of the mushroom,' said the Caterpillar, just as if she had +asked it aloud; and in another moment it was out of sight. + + Alice remained looking thoughtfully at the mushroom for a +minute, trying to make out which were the two sides of it; and as +it was perfectly round, she found this a very difficult question. +However, at last she stretched her arms round it as far as they +would go, and broke off a bit of the edge with each hand. + + `And now which is which?' she said to herself, and nibbled a +little of the right-hand bit to try the effect: the next moment +she felt a violent blow underneath her chin: it had struck her +foot! + + She was a good deal frightened by this very sudden change, but +she felt that there was no time to be lost, as she was shrinking +rapidly; so she set to work at once to eat some of the other bit. +Her chin was pressed so closely against her foot, that there was +hardly room to open her mouth; but she did it at last, and +managed to swallow a morsel of the lefthand bit. + + + * * * * * * * + + * * * * * * + + * * * * * * * + + `Come, my head's free at last!' said Alice in a tone of +delight, which changed into alarm in another moment, when she +found that her shoulders were nowhere to be found: all she could +see, when she looked down, was an immense length of neck, which +seemed to rise like a stalk out of a sea of green leaves that lay +far below her. + + `What CAN all that green stuff be?' said Alice. `And where +HAVE my shoulders got to? And oh, my poor hands, how is it I +can't see you?' She was moving them about as she spoke, but no +result seemed to follow, except a little shaking among the +distant green leaves. + + As there seemed to be no chance of getting her hands up to her +head, she tried to get her head down to them, and was delighted +to find that her neck would bend about easily in any direction, +like a serpent. She had just succeeded in curving it down into a +graceful zigzag, and was going to dive in among the leaves, which +she found to be nothing but the tops of the trees under which she +had been wandering, when a sharp hiss made her draw back in a +hurry: a large pigeon had flown into her face, and was beating +her violently with its wings. + + `Serpent!' screamed the Pigeon. + + `I'm NOT a serpent!' said Alice indignantly. `Let me alone!' + + `Serpent, I say again!' repeated the Pigeon, but in a more +subdued tone, and added with a kind of sob, `I've tried every +way, and nothing seems to suit them!' + + `I haven't the least idea what you're talking about,' said +Alice. + + `I've tried the roots of trees, and I've tried banks, and I've +tried hedges,' the Pigeon went on, without attending to her; `but +those serpents! There's no pleasing them!' + + Alice was more and more puzzled, but she thought there was no +use in saying anything more till the Pigeon had finished. + + `As if it wasn't trouble enough hatching the eggs,' said the +Pigeon; `but I must be on the look-out for serpents night and +day! Why, I haven't had a wink of sleep these three weeks!' + + `I'm very sorry you've been annoyed,' said Alice, who was +beginning to see its meaning. + + `And just as I'd taken the highest tree in the wood,' continued +the Pigeon, raising its voice to a shriek, `and just as I was +thinking I should be free of them at last, they must needs come +wriggling down from the sky! Ugh, Serpent!' + + `But I'm NOT a serpent, I tell you!' said Alice. `I'm a--I'm +a--' + + `Well! WHAT are you?' said the Pigeon. `I can see you're +trying to invent something!' + + `I--I'm a little girl,' said Alice, rather doubtfully, as she +remembered the number of changes she had gone through that day. + + `A likely story indeed!' said the Pigeon in a tone of the +deepest contempt. `I've seen a good many little girls in my +time, but never ONE with such a neck as that! No, no! You're a +serpent; and there's no use denying it. I suppose you'll be +telling me next that you never tasted an egg!' + + `I HAVE tasted eggs, certainly,' said Alice, who was a very +truthful child; `but little girls eat eggs quite as much as +serpents do, you know.' + + `I don't believe it,' said the Pigeon; `but if they do, why +then they're a kind of serpent, that's all I can say.' + + This was such a new idea to Alice, that she was quite silent +for a minute or two, which gave the Pigeon the opportunity of +adding, `You're looking for eggs, I know THAT well enough; and +what does it matter to me whether you're a little girl or a +serpent?' + + `It matters a good deal to ME,' said Alice hastily; `but I'm +not looking for eggs, as it happens; and if I was, I shouldn't +want YOURS: I don't like them raw.' + + `Well, be off, then!' said the Pigeon in a sulky tone, as it +settled down again into its nest. Alice crouched down among the +trees as well as she could, for her neck kept getting entangled +among the branches, and every now and then she had to stop and +untwist it. After a while she remembered that she still held the +pieces of mushroom in her hands, and she set to work very +carefully, nibbling first at one and then at the other, and +growing sometimes taller and sometimes shorter, until she had +succeeded in bringing herself down to her usual height. + + It was so long since she had been anything near the right size, +that it felt quite strange at first; but she got used to it in a +few minutes, and began talking to herself, as usual. `Come, +there's half my plan done now! How puzzling all these changes +are! I'm never sure what I'm going to be, from one minute to +another! However, I've got back to my right size: the next +thing is, to get into that beautiful garden--how IS that to be +done, I wonder?' As she said this, she came suddenly upon an +open place, with a little house in it about four feet high. +`Whoever lives there,' thought Alice, `it'll never do to come +upon them THIS size: why, I should frighten them out of their +wits!' So she began nibbling at the righthand bit again, and did +not venture to go near the house till she had brought herself +down to nine inches high. + + + + CHAPTER VI + + Pig and Pepper + + + For a minute or two she stood looking at the house, and +wondering what to do next, when suddenly a footman in livery came +running out of the wood--(she considered him to be a footman +because he was in livery: otherwise, judging by his face only, +she would have called him a fish)--and rapped loudly at the door +with his knuckles. It was opened by another footman in livery, +with a round face, and large eyes like a frog; and both footmen, +Alice noticed, had powdered hair that curled all over their +heads. She felt very curious to know what it was all about, and +crept a little way out of the wood to listen. + + The Fish-Footman began by producing from under his arm a great +letter, nearly as large as himself, and this he handed over to +the other, saying, in a solemn tone, `For the Duchess. An +invitation from the Queen to play croquet.' The Frog-Footman +repeated, in the same solemn tone, only changing the order of the +words a little, `From the Queen. An invitation for the Duchess +to play croquet.' + + Then they both bowed low, and their curls got entangled +together. + + Alice laughed so much at this, that she had to run back into +the wood for fear of their hearing her; and when she next peeped +out the Fish-Footman was gone, and the other was sitting on the +ground near the door, staring stupidly up into the sky. + + Alice went timidly up to the door, and knocked. + + `There's no sort of use in knocking,' said the Footman, `and +that for two reasons. First, because I'm on the same side of the +door as you are; secondly, because they're making such a noise +inside, no one could possibly hear you.' And certainly there was +a most extraordinary noise going on within--a constant howling +and sneezing, and every now and then a great crash, as if a dish +or kettle had been broken to pieces. + + `Please, then,' said Alice, `how am I to get in?' + + `There might be some sense in your knocking,' the Footman went +on without attending to her, `if we had the door between us. For +instance, if you were INSIDE, you might knock, and I could let +you out, you know.' He was looking up into the sky all the time +he was speaking, and this Alice thought decidedly uncivil. `But +perhaps he can't help it,' she said to herself; `his eyes are so +VERY nearly at the top of his head. But at any rate he might +answer questions.--How am I to get in?' she repeated, aloud. + + `I shall sit here,' the Footman remarked, `till tomorrow--' + + At this moment the door of the house opened, and a large plate +came skimming out, straight at the Footman's head: it just +grazed his nose, and broke to pieces against one of the trees +behind him. + + `--or next day, maybe,' the Footman continued in the same tone, +exactly as if nothing had happened. + + `How am I to get in?' asked Alice again, in a louder tone. + + `ARE you to get in at all?' said the Footman. `That's the +first question, you know.' + + It was, no doubt: only Alice did not like to be told so. +`It's really dreadful,' she muttered to herself, `the way all the +creatures argue. It's enough to drive one crazy!' + + The Footman seemed to think this a good opportunity for +repeating his remark, with variations. `I shall sit here,' he +said, `on and off, for days and days.' + + `But what am I to do?' said Alice. + + `Anything you like,' said the Footman, and began whistling. + + `Oh, there's no use in talking to him,' said Alice desperately: +`he's perfectly idiotic!' And she opened the door and went in. + + The door led right into a large kitchen, which was full of +smoke from one end to the other: the Duchess was sitting on a +three-legged stool in the middle, nursing a baby; the cook was +leaning over the fire, stirring a large cauldron which seemed to +be full of soup. + + `There's certainly too much pepper in that soup!' Alice said to +herself, as well as she could for sneezing. + + There was certainly too much of it in the air. Even the +Duchess sneezed occasionally; and as for the baby, it was +sneezing and howling alternately without a moment's pause. The +only things in the kitchen that did not sneeze, were the cook, +and a large cat which was sitting on the hearth and grinning from +ear to ear. + + `Please would you tell me,' said Alice, a little timidly, for +she was not quite sure whether it was good manners for her to +speak first, `why your cat grins like that?' + + `It's a Cheshire cat,' said the Duchess, `and that's why. +Pig!' + + She said the last word with such sudden violence that Alice +quite jumped; but she saw in another moment that it was addressed +to the baby, and not to her, so she took courage, and went on +again:-- + + `I didn't know that Cheshire cats always grinned; in fact, I +didn't know that cats COULD grin.' + + `They all can,' said the Duchess; `and most of 'em do.' + + `I don't know of any that do,' Alice said very politely, +feeling quite pleased to have got into a conversation. + + `You don't know much,' said the Duchess; `and that's a fact.' + + Alice did not at all like the tone of this remark, and thought +it would be as well to introduce some other subject of +conversation. While she was trying to fix on one, the cook took +the cauldron of soup off the fire, and at once set to work +throwing everything within her reach at the Duchess and the baby +--the fire-irons came first; then followed a shower of saucepans, +plates, and dishes. The Duchess took no notice of them even when +they hit her; and the baby was howling so much already, that it +was quite impossible to say whether the blows hurt it or not. + + `Oh, PLEASE mind what you're doing!' cried Alice, jumping up +and down in an agony of terror. `Oh, there goes his PRECIOUS +nose'; as an unusually large saucepan flew close by it, and very +nearly carried it off. + + `If everybody minded their own business,' the Duchess said in a +hoarse growl, `the world would go round a deal faster than it +does.' + + `Which would NOT be an advantage,' said Alice, who felt very +glad to get an opportunity of showing off a little of her +knowledge. `Just think of what work it would make with the day +and night! You see the earth takes twenty-four hours to turn +round on its axis--' + + `Talking of axes,' said the Duchess, `chop off her head!' + + Alice glanced rather anxiously at the cook, to see if she meant +to take the hint; but the cook was busily stirring the soup, and +seemed not to be listening, so she went on again: `Twenty-four +hours, I THINK; or is it twelve? I--' + + `Oh, don't bother ME,' said the Duchess; `I never could abide +figures!' And with that she began nursing her child again, +singing a sort of lullaby to it as she did so, and giving it a +violent shake at the end of every line: + + `Speak roughly to your little boy, + And beat him when he sneezes: + He only does it to annoy, + Because he knows it teases.' + + CHORUS. + + (In which the cook and the baby joined):-- + + `Wow! wow! wow!' + + While the Duchess sang the second verse of the song, she kept +tossing the baby violently up and down, and the poor little thing +howled so, that Alice could hardly hear the words:-- + + `I speak severely to my boy, + I beat him when he sneezes; + For he can thoroughly enjoy + The pepper when he pleases!' + + CHORUS. + + `Wow! wow! wow!' + + `Here! you may nurse it a bit, if you like!' the Duchess said +to Alice, flinging the baby at her as she spoke. `I must go and +get ready to play croquet with the Queen,' and she hurried out of +the room. The cook threw a frying-pan after her as she went out, +but it just missed her. + + Alice caught the baby with some difficulty, as it was a queer- +shaped little creature, and held out its arms and legs in all +directions, `just like a star-fish,' thought Alice. The poor +little thing was snorting like a steam-engine when she caught it, +and kept doubling itself up and straightening itself out again, +so that altogether, for the first minute or two, it was as much +as she could do to hold it. + + As soon as she had made out the proper way of nursing it, +(which was to twist it up into a sort of knot, and then keep +tight hold of its right ear and left foot, so as to prevent its +undoing itself,) she carried it out into the open air. `IF I +don't take this child away with me,' thought Alice, `they're sure +to kill it in a day or two: wouldn't it be murder to leave it +behind?' She said the last words out loud, and the little thing +grunted in reply (it had left off sneezing by this time). `Don't +grunt,' said Alice; `that's not at all a proper way of expressing +yourself.' + + The baby grunted again, and Alice looked very anxiously into +its face to see what was the matter with it. There could be no +doubt that it had a VERY turn-up nose, much more like a snout +than a real nose; also its eyes were getting extremely small for +a baby: altogether Alice did not like the look of the thing at +all. `But perhaps it was only sobbing,' she thought, and looked +into its eyes again, to see if there were any tears. + + No, there were no tears. `If you're going to turn into a pig, +my dear,' said Alice, seriously, `I'll have nothing more to do +with you. Mind now!' The poor little thing sobbed again (or +grunted, it was impossible to say which), and they went on for +some while in silence. + + Alice was just beginning to think to herself, `Now, what am I +to do with this creature when I get it home?' when it grunted +again, so violently, that she looked down into its face in some +alarm. This time there could be NO mistake about it: it was +neither more nor less than a pig, and she felt that it would be +quite absurd for her to carry it further. + + So she set the little creature down, and felt quite relieved to +see it trot away quietly into the wood. `If it had grown up,' +she said to herself, `it would have made a dreadfully ugly child: +but it makes rather a handsome pig, I think.' And she began +thinking over other children she knew, who might do very well as +pigs, and was just saying to herself, `if one only knew the right +way to change them--' when she was a little startled by seeing +the Cheshire Cat sitting on a bough of a tree a few yards off. + + The Cat only grinned when it saw Alice. It looked good- +natured, she thought: still it had VERY long claws and a great +many teeth, so she felt that it ought to be treated with respect. + + `Cheshire Puss,' she began, rather timidly, as she did not at +all know whether it would like the name: however, it only +grinned a little wider. `Come, it's pleased so far,' thought +Alice, and she went on. `Would you tell me, please, which way I +ought to go from here?' + + `That depends a good deal on where you want to get to,' said +the Cat. + + `I don't much care where--' said Alice. + + `Then it doesn't matter which way you go,' said the Cat. + + `--so long as I get SOMEWHERE,' Alice added as an explanation. + + `Oh, you're sure to do that,' said the Cat, `if you only walk +long enough.' + + Alice felt that this could not be denied, so she tried another +question. `What sort of people live about here?' + + `In THAT direction,' the Cat said, waving its right paw round, +`lives a Hatter: and in THAT direction,' waving the other paw, +`lives a March Hare. Visit either you like: they're both mad.' + + `But I don't want to go among mad people,' Alice remarked. + + `Oh, you can't help that,' said the Cat: `we're all mad here. +I'm mad. You're mad.' + + `How do you know I'm mad?' said Alice. + + `You must be,' said the Cat, `or you wouldn't have come here.' + + Alice didn't think that proved it at all; however, she went on +`And how do you know that you're mad?' + + `To begin with,' said the Cat, `a dog's not mad. You grant +that?' + + `I suppose so,' said Alice. + + `Well, then,' the Cat went on, `you see, a dog growls when it's +angry, and wags its tail when it's pleased. Now I growl when I'm +pleased, and wag my tail when I'm angry. Therefore I'm mad.' + + `I call it purring, not growling,' said Alice. + + `Call it what you like,' said the Cat. `Do you play croquet +with the Queen to-day?' + + `I should like it very much,' said Alice, `but I haven't been +invited yet.' + + `You'll see me there,' said the Cat, and vanished. + + Alice was not much surprised at this, she was getting so used +to queer things happening. While she was looking at the place +where it had been, it suddenly appeared again. + + `By-the-bye, what became of the baby?' said the Cat. `I'd +nearly forgotten to ask.' + + `It turned into a pig,' Alice quietly said, just as if it had +come back in a natural way. + + `I thought it would,' said the Cat, and vanished again. + + Alice waited a little, half expecting to see it again, but it +did not appear, and after a minute or two she walked on in the +direction in which the March Hare was said to live. `I've seen +hatters before,' she said to herself; `the March Hare will be +much the most interesting, and perhaps as this is May it won't be +raving mad--at least not so mad as it was in March.' As she said +this, she looked up, and there was the Cat again, sitting on a +branch of a tree. + + `Did you say pig, or fig?' said the Cat. + + `I said pig,' replied Alice; `and I wish you wouldn't keep +appearing and vanishing so suddenly: you make one quite giddy.' + + `All right,' said the Cat; and this time it vanished quite +slowly, beginning with the end of the tail, and ending with the +grin, which remained some time after the rest of it had gone. + + `Well! I've often seen a cat without a grin,' thought Alice; +`but a grin without a cat! It's the most curious thing I ever +say in my life!' + + She had not gone much farther before she came in sight of the +house of the March Hare: she thought it must be the right house, +because the chimneys were shaped like ears and the roof was +thatched with fur. It was so large a house, that she did not +like to go nearer till she had nibbled some more of the lefthand +bit of mushroom, and raised herself to about two feet high: even +then she walked up towards it rather timidly, saying to herself +`Suppose it should be raving mad after all! I almost wish I'd +gone to see the Hatter instead!' + + + + CHAPTER VII + + A Mad Tea-Party + + + There was a table set out under a tree in front of the house, +and the March Hare and the Hatter were having tea at it: a +Dormouse was sitting between them, fast asleep, and the other two +were using it as a cushion, resting their elbows on it, and the +talking over its head. `Very uncomfortable for the Dormouse,' +thought Alice; `only, as it's asleep, I suppose it doesn't mind.' + + The table was a large one, but the three were all crowded +together at one corner of it: `No room! No room!' they cried +out when they saw Alice coming. `There's PLENTY of room!' said +Alice indignantly, and she sat down in a large arm-chair at one +end of the table. + + `Have some wine,' the March Hare said in an encouraging tone. + + Alice looked all round the table, but there was nothing on it +but tea. `I don't see any wine,' she remarked. + + `There isn't any,' said the March Hare. + + `Then it wasn't very civil of you to offer it,' said Alice +angrily. + + `It wasn't very civil of you to sit down without being +invited,' said the March Hare. + + `I didn't know it was YOUR table,' said Alice; `it's laid for a +great many more than three.' + + `Your hair wants cutting,' said the Hatter. He had been +looking at Alice for some time with great curiosity, and this was +his first speech. + + `You should learn not to make personal remarks,' Alice said +with some severity; `it's very rude.' + + The Hatter opened his eyes very wide on hearing this; but all +he SAID was, `Why is a raven like a writing-desk?' + + `Come, we shall have some fun now!' thought Alice. `I'm glad +they've begun asking riddles.--I believe I can guess that,' she +added aloud. + + `Do you mean that you think you can find out the answer to it?' +said the March Hare. + + `Exactly so,' said Alice. + + `Then you should say what you mean,' the March Hare went on. + + `I do,' Alice hastily replied; `at least--at least I mean what +I say--that's the same thing, you know.' + + `Not the same thing a bit!' said the Hatter. `You might just +as well say that "I see what I eat" is the same thing as "I eat +what I see"!' + + `You might just as well say,' added the March Hare, `that "I +like what I get" is the same thing as "I get what I like"!' + + `You might just as well say,' added the Dormouse, who seemed to +be talking in his sleep, `that "I breathe when I sleep" is the +same thing as "I sleep when I breathe"!' + + `It IS the same thing with you,' said the Hatter, and here the +conversation dropped, and the party sat silent for a minute, +while Alice thought over all she could remember about ravens and +writing-desks, which wasn't much. + + The Hatter was the first to break the silence. `What day of +the month is it?' he said, turning to Alice: he had taken his +watch out of his pocket, and was looking at it uneasily, shaking +it every now and then, and holding it to his ear. + + Alice considered a little, and then said `The fourth.' + + `Two days wrong!' sighed the Hatter. `I told you butter +wouldn't suit the works!' he added looking angrily at the March +Hare. + + `It was the BEST butter,' the March Hare meekly replied. + + `Yes, but some crumbs must have got in as well,' the Hatter +grumbled: `you shouldn't have put it in with the bread-knife.' + + The March Hare took the watch and looked at it gloomily: then +he dipped it into his cup of tea, and looked at it again: but he +could think of nothing better to say than his first remark, `It +was the BEST butter, you know.' + + Alice had been looking over his shoulder with some curiosity. +`What a funny watch!' she remarked. `It tells the day of the +month, and doesn't tell what o'clock it is!' + + `Why should it?' muttered the Hatter. `Does YOUR watch tell +you what year it is?' + + `Of course not,' Alice replied very readily: `but that's +because it stays the same year for such a long time together.' + + `Which is just the case with MINE,' said the Hatter. + + Alice felt dreadfully puzzled. The Hatter's remark seemed to +have no sort of meaning in it, and yet it was certainly English. +`I don't quite understand you,' she said, as politely as she +could. + + `The Dormouse is asleep again,' said the Hatter, and he poured +a little hot tea upon its nose. + + The Dormouse shook its head impatiently, and said, without +opening its eyes, `Of course, of course; just what I was going to +remark myself.' + + `Have you guessed the riddle yet?' the Hatter said, turning to +Alice again. + + `No, I give it up,' Alice replied: `what's the answer?' + + `I haven't the slightest idea,' said the Hatter. + + `Nor I,' said the March Hare. + + Alice sighed wearily. `I think you might do something better +with the time,' she said, `than waste it in asking riddles that +have no answers.' + + `If you knew Time as well as I do,' said the Hatter, `you +wouldn't talk about wasting IT. It's HIM.' + + `I don't know what you mean,' said Alice. + + `Of course you don't!' the Hatter said, tossing his head +contemptuously. `I dare say you never even spoke to Time!' + + `Perhaps not,' Alice cautiously replied: `but I know I have to +beat time when I learn music.' + + `Ah! that accounts for it,' said the Hatter. `He won't stand +beating. Now, if you only kept on good terms with him, he'd do +almost anything you liked with the clock. For instance, suppose +it were nine o'clock in the morning, just time to begin lessons: +you'd only have to whisper a hint to Time, and round goes the +clock in a twinkling! Half-past one, time for dinner!' + + (`I only wish it was,' the March Hare said to itself in a +whisper.) + + `That would be grand, certainly,' said Alice thoughtfully: +`but then--I shouldn't be hungry for it, you know.' + + `Not at first, perhaps,' said the Hatter: `but you could keep +it to half-past one as long as you liked.' + + `Is that the way YOU manage?' Alice asked. + + The Hatter shook his head mournfully. `Not I!' he replied. +`We quarrelled last March--just before HE went mad, you know--' +(pointing with his tea spoon at the March Hare,) `--it was at the +great concert given by the Queen of Hearts, and I had to sing + + "Twinkle, twinkle, little bat! + How I wonder what you're at!" + +You know the song, perhaps?' + + `I've heard something like it,' said Alice. + + `It goes on, you know,' the Hatter continued, `in this way:-- + + "Up above the world you fly, + Like a tea-tray in the sky. + Twinkle, twinkle--"' + +Here the Dormouse shook itself, and began singing in its sleep +`Twinkle, twinkle, twinkle, twinkle--' and went on so long that +they had to pinch it to make it stop. + + `Well, I'd hardly finished the first verse,' said the Hatter, +`when the Queen jumped up and bawled out, "He's murdering the +time! Off with his head!"' + + `How dreadfully savage!' exclaimed Alice. + + `And ever since that,' the Hatter went on in a mournful tone, +`he won't do a thing I ask! It's always six o'clock now.' + + A bright idea came into Alice's head. `Is that the reason so +many tea-things are put out here?' she asked. + + `Yes, that's it,' said the Hatter with a sigh: `it's always +tea-time, and we've no time to wash the things between whiles.' + + `Then you keep moving round, I suppose?' said Alice. + + `Exactly so,' said the Hatter: `as the things get used up.' + + `But what happens when you come to the beginning again?' Alice +ventured to ask. + + `Suppose we change the subject,' the March Hare interrupted, +yawning. `I'm getting tired of this. I vote the young lady +tells us a story.' + + `I'm afraid I don't know one,' said Alice, rather alarmed at +the proposal. + + `Then the Dormouse shall!' they both cried. `Wake up, +Dormouse!' And they pinched it on both sides at once. + + The Dormouse slowly opened his eyes. `I wasn't asleep,' he +said in a hoarse, feeble voice: `I heard every word you fellows +were saying.' + + `Tell us a story!' said the March Hare. + + `Yes, please do!' pleaded Alice. + + `And be quick about it,' added the Hatter, `or you'll be asleep +again before it's done.' + + `Once upon a time there were three little sisters,' the +Dormouse began in a great hurry; `and their names were Elsie, +Lacie, and Tillie; and they lived at the bottom of a well--' + + `What did they live on?' said Alice, who always took a great +interest in questions of eating and drinking. + + `They lived on treacle,' said the Dormouse, after thinking a +minute or two. + + `They couldn't have done that, you know,' Alice gently +remarked; `they'd have been ill.' + + `So they were,' said the Dormouse; `VERY ill.' + + Alice tried to fancy to herself what such an extraordinary ways +of living would be like, but it puzzled her too much, so she went +on: `But why did they live at the bottom of a well?' + + `Take some more tea,' the March Hare said to Alice, very +earnestly. + + `I've had nothing yet,' Alice replied in an offended tone, `so +I can't take more.' + + `You mean you can't take LESS,' said the Hatter: `it's very +easy to take MORE than nothing.' + + `Nobody asked YOUR opinion,' said Alice. + + `Who's making personal remarks now?' the Hatter asked +triumphantly. + + Alice did not quite know what to say to this: so she helped +herself to some tea and bread-and-butter, and then turned to the +Dormouse, and repeated her question. `Why did they live at the +bottom of a well?' + + The Dormouse again took a minute or two to think about it, and +then said, `It was a treacle-well.' + + `There's no such thing!' Alice was beginning very angrily, but +the Hatter and the March Hare went `Sh! sh!' and the Dormouse +sulkily remarked, `If you can't be civil, you'd better finish the +story for yourself.' + + `No, please go on!' Alice said very humbly; `I won't interrupt +again. I dare say there may be ONE.' + + `One, indeed!' said the Dormouse indignantly. However, he +consented to go on. `And so these three little sisters--they +were learning to draw, you know--' + + `What did they draw?' said Alice, quite forgetting her promise. + + `Treacle,' said the Dormouse, without considering at all this +time. + + `I want a clean cup,' interrupted the Hatter: `let's all move +one place on.' + + He moved on as he spoke, and the Dormouse followed him: the +March Hare moved into the Dormouse's place, and Alice rather +unwillingly took the place of the March Hare. The Hatter was the +only one who got any advantage from the change: and Alice was a +good deal worse off than before, as the March Hare had just upset +the milk-jug into his plate. + + Alice did not wish to offend the Dormouse again, so she began +very cautiously: `But I don't understand. Where did they draw +the treacle from?' + + `You can draw water out of a water-well,' said the Hatter; `so +I should think you could draw treacle out of a treacle-well--eh, +stupid?' + + `But they were IN the well,' Alice said to the Dormouse, not +choosing to notice this last remark. + + `Of course they were', said the Dormouse; `--well in.' + + This answer so confused poor Alice, that she let the Dormouse +go on for some time without interrupting it. + + `They were learning to draw,' the Dormouse went on, yawning and +rubbing its eyes, for it was getting very sleepy; `and they drew +all manner of things--everything that begins with an M--' + + `Why with an M?' said Alice. + + `Why not?' said the March Hare. + + Alice was silent. + + The Dormouse had closed its eyes by this time, and was going +off into a doze; but, on being pinched by the Hatter, it woke up +again with a little shriek, and went on: `--that begins with an +M, such as mouse-traps, and the moon, and memory, and muchness-- +you know you say things are "much of a muchness"--did you ever +see such a thing as a drawing of a muchness?' + + `Really, now you ask me,' said Alice, very much confused, `I +don't think--' + + `Then you shouldn't talk,' said the Hatter. + + This piece of rudeness was more than Alice could bear: she got +up in great disgust, and walked off; the Dormouse fell asleep +instantly, and neither of the others took the least notice of her +going, though she looked back once or twice, half hoping that +they would call after her: the last time she saw them, they were +trying to put the Dormouse into the teapot. + + `At any rate I'll never go THERE again!' said Alice as she +picked her way through the wood. `It's the stupidest tea-party I +ever was at in all my life!' + + Just as she said this, she noticed that one of the trees had a +door leading right into it. `That's very curious!' she thought. +`But everything's curious today. I think I may as well go in at +once.' And in she went. + + Once more she found herself in the long hall, and close to the +little glass table. `Now, I'll manage better this time,' she +said to herself, and began by taking the little golden key, and +unlocking the door that led into the garden. Then she went to +work nibbling at the mushroom (she had kept a piece of it in her +pocked) till she was about a foot high: then she walked down the +little passage: and THEN--she found herself at last in the +beautiful garden, among the bright flower-beds and the cool +fountains. + + + + CHAPTER VIII + + The Queen's Croquet-Ground + + + A large rose-tree stood near the entrance of the garden: the +roses growing on it were white, but there were three gardeners at +it, busily painting them red. Alice thought this a very curious +thing, and she went nearer to watch them, and just as she came up +to them she heard one of them say, `Look out now, Five! Don't go +splashing paint over me like that!' + + `I couldn't help it,' said Five, in a sulky tone; `Seven jogged +my elbow.' + + On which Seven looked up and said, `That's right, Five! Always +lay the blame on others!' + + `YOU'D better not talk!' said Five. `I heard the Queen say only +yesterday you deserved to be beheaded!' + + `What for?' said the one who had spoken first. + + `That's none of YOUR business, Two!' said Seven. + + `Yes, it IS his business!' said Five, `and I'll tell him--it +was for bringing the cook tulip-roots instead of onions.' + + Seven flung down his brush, and had just begun `Well, of all +the unjust things--' when his eye chanced to fall upon Alice, as +she stood watching them, and he checked himself suddenly: the +others looked round also, and all of them bowed low. + + `Would you tell me,' said Alice, a little timidly, `why you are +painting those roses?' + + Five and Seven said nothing, but looked at Two. Two began in a +low voice, `Why the fact is, you see, Miss, this here ought to +have been a RED rose-tree, and we put a white one in by mistake; +and if the Queen was to find it out, we should all have our heads +cut off, you know. So you see, Miss, we're doing our best, afore +she comes, to--' At this moment Five, who had been anxiously +looking across the garden, called out `The Queen! The Queen!' +and the three gardeners instantly threw themselves flat upon +their faces. There was a sound of many footsteps, and Alice +looked round, eager to see the Queen. + + First came ten soldiers carrying clubs; these were all shaped +like the three gardeners, oblong and flat, with their hands and +feet at the corners: next the ten courtiers; these were +ornamented all over with diamonds, and walked two and two, as the +soldiers did. After these came the royal children; there were +ten of them, and the little dears came jumping merrily along hand +in hand, in couples: they were all ornamented with hearts. Next +came the guests, mostly Kings and Queens, and among them Alice +recognised the White Rabbit: it was talking in a hurried nervous +manner, smiling at everything that was said, and went by without +noticing her. Then followed the Knave of Hearts, carrying the +King's crown on a crimson velvet cushion; and, last of all this +grand procession, came THE KING AND QUEEN OF HEARTS. + + Alice was rather doubtful whether she ought not to lie down on +her face like the three gardeners, but she could not remember +every having heard of such a rule at processions; `and besides, +what would be the use of a procession,' thought she, `if people +had all to lie down upon their faces, so that they couldn't see +it?' So she stood still where she was, and waited. + + When the procession came opposite to Alice, they all stopped +and looked at her, and the Queen said severely `Who is this?' +She said it to the Knave of Hearts, who only bowed and smiled in +reply. + + `Idiot!' said the Queen, tossing her head impatiently; and, +turning to Alice, she went on, `What's your name, child?' + + `My name is Alice, so please your Majesty,' said Alice very +politely; but she added, to herself, `Why, they're only a pack of +cards, after all. I needn't be afraid of them!' + + `And who are THESE?' said the Queen, pointing to the three +gardeners who were lying round the rosetree; for, you see, as +they were lying on their faces, and the pattern on their backs +was the same as the rest of the pack, she could not tell whether +they were gardeners, or soldiers, or courtiers, or three of her +own children. + + `How should I know?' said Alice, surprised at her own courage. +`It's no business of MINE.' + + The Queen turned crimson with fury, and, after glaring at her +for a moment like a wild beast, screamed `Off with her head! +Off--' + + `Nonsense!' said Alice, very loudly and decidedly, and the +Queen was silent. + + The King laid his hand upon her arm, and timidly said +`Consider, my dear: she is only a child!' + + The Queen turned angrily away from him, and said to the Knave +`Turn them over!' + + The Knave did so, very carefully, with one foot. + + `Get up!' said the Queen, in a shrill, loud voice, and the +three gardeners instantly jumped up, and began bowing to the +King, the Queen, the royal children, and everybody else. + + `Leave off that!' screamed the Queen. `You make me giddy.' +And then, turning to the rose-tree, she went on, `What HAVE you +been doing here?' + + `May it please your Majesty,' said Two, in a very humble tone, +going down on one knee as he spoke, `we were trying--' + + `I see!' said the Queen, who had meanwhile been examining the +roses. `Off with their heads!' and the procession moved on, +three of the soldiers remaining behind to execute the unfortunate +gardeners, who ran to Alice for protection. + + `You shan't be beheaded!' said Alice, and she put them into a +large flower-pot that stood near. The three soldiers wandered +about for a minute or two, looking for them, and then quietly +marched off after the others. + + `Are their heads off?' shouted the Queen. + + `Their heads are gone, if it please your Majesty!' the soldiers +shouted in reply. + + `That's right!' shouted the Queen. `Can you play croquet?' + + The soldiers were silent, and looked at Alice, as the question +was evidently meant for her. + + `Yes!' shouted Alice. + + `Come on, then!' roared the Queen, and Alice joined the +procession, wondering very much what would happen next. + + `It's--it's a very fine day!' said a timid voice at her side. +She was walking by the White Rabbit, who was peeping anxiously +into her face. + + `Very,' said Alice: `--where's the Duchess?' + + `Hush! Hush!' said the Rabbit in a low, hurried tone. He +looked anxiously over his shoulder as he spoke, and then raised +himself upon tiptoe, put his mouth close to her ear, and +whispered `She's under sentence of execution.' + + `What for?' said Alice. + + `Did you say "What a pity!"?' the Rabbit asked. + + `No, I didn't,' said Alice: `I don't think it's at all a pity. +I said "What for?"' + + `She boxed the Queen's ears--' the Rabbit began. Alice gave a +little scream of laughter. `Oh, hush!' the Rabbit whispered in a +frightened tone. `The Queen will hear you! You see, she came +rather late, and the Queen said--' + + `Get to your places!' shouted the Queen in a voice of thunder, +and people began running about in all directions, tumbling up +against each other; however, they got settled down in a minute or +two, and the game began. Alice thought she had never seen such a +curious croquet-ground in her life; it was all ridges and +furrows; the balls were live hedgehogs, the mallets live +flamingoes, and the soldiers had to double themselves up and to +stand on their hands and feet, to make the arches. + + The chief difficulty Alice found at first was in managing her +flamingo: she succeeded in getting its body tucked away, +comfortably enough, under her arm, with its legs hanging down, +but generally, just as she had got its neck nicely straightened +out, and was going to give the hedgehog a blow with its head, it +WOULD twist itself round and look up in her face, with such a +puzzled expression that she could not help bursting out laughing: +and when she had got its head down, and was going to begin again, +it was very provoking to find that the hedgehog had unrolled +itself, and was in the act of crawling away: besides all this, +there was generally a ridge or furrow in the way wherever she +wanted to send the hedgehog to, and, as the doubled-up soldiers +were always getting up and walking off to other parts of the +ground, Alice soon came to the conclusion that it was a very +difficult game indeed. + + The players all played at once without waiting for turns, +quarrelling all the while, and fighting for the hedgehogs; and in +a very short time the Queen was in a furious passion, and went +stamping about, and shouting `Off with his head!' or `Off with +her head!' about once in a minute. + + Alice began to feel very uneasy: to be sure, she had not as +yet had any dispute with the Queen, but she knew that it might +happen any minute, `and then,' thought she, `what would become of +me? They're dreadfully fond of beheading people here; the great +wonder is, that there's any one left alive!' + + She was looking about for some way of escape, and wondering +whether she could get away without being seen, when she noticed a +curious appearance in the air: it puzzled her very much at +first, but, after watching it a minute or two, she made it out to +be a grin, and she said to herself `It's the Cheshire Cat: now I +shall have somebody to talk to.' + + `How are you getting on?' said the Cat, as soon as there was +mouth enough for it to speak with. + + Alice waited till the eyes appeared, and then nodded. `It's no +use speaking to it,' she thought, `till its ears have come, or at +least one of them.' In another minute the whole head appeared, +and then Alice put down her flamingo, and began an account of the +game, feeling very glad she had someone to listen to her. The +Cat seemed to think that there was enough of it now in sight, and +no more of it appeared. + + `I don't think they play at all fairly,' Alice began, in rather +a complaining tone, `and they all quarrel so dreadfully one can't +hear oneself speak--and they don't seem to have any rules in +particular; at least, if there are, nobody attends to them--and +you've no idea how confusing it is all the things being alive; +for instance, there's the arch I've got to go through next +walking about at the other end of the ground--and I should have +croqueted the Queen's hedgehog just now, only it ran away when it +saw mine coming!' + + `How do you like the Queen?' said the Cat in a low voice. + + `Not at all,' said Alice: `she's so extremely--' Just then +she noticed that the Queen was close behind her, listening: so +she went on, `--likely to win, that it's hardly worth while +finishing the game.' + + The Queen smiled and passed on. + + `Who ARE you talking to?' said the King, going up to Alice, and +looking at the Cat's head with great curiosity. + + `It's a friend of mine--a Cheshire Cat,' said Alice: `allow me +to introduce it.' + + `I don't like the look of it at all,' said the King: `however, +it may kiss my hand if it likes.' + + `I'd rather not,' the Cat remarked. + + `Don't be impertinent,' said the King, `and don't look at me +like that!' He got behind Alice as he spoke. + + `A cat may look at a king,' said Alice. `I've read that in +some book, but I don't remember where.' + + `Well, it must be removed,' said the King very decidedly, and +he called the Queen, who was passing at the moment, `My dear! I +wish you would have this cat removed!' + + The Queen had only one way of settling all difficulties, great +or small. `Off with his head!' she said, without even looking +round. + + `I'll fetch the executioner myself,' said the King eagerly, and +he hurried off. + + Alice thought she might as well go back, and see how the game +was going on, as she heard the Queen's voice in the distance, +screaming with passion. She had already heard her sentence three +of the players to be executed for having missed their turns, and +she did not like the look of things at all, as the game was in +such confusion that she never knew whether it was her turn or +not. So she went in search of her hedgehog. + + The hedgehog was engaged in a fight with another hedgehog, +which seemed to Alice an excellent opportunity for croqueting one +of them with the other: the only difficulty was, that her +flamingo was gone across to the other side of the garden, where +Alice could see it trying in a helpless sort of way to fly up +into a tree. + + By the time she had caught the flamingo and brought it back, +the fight was over, and both the hedgehogs were out of sight: +`but it doesn't matter much,' thought Alice, `as all the arches +are gone from this side of the ground.' So she tucked it away +under her arm, that it might not escape again, and went back for +a little more conversation with her friend. + + When she got back to the Cheshire Cat, she was surprised to +find quite a large crowd collected round it: there was a dispute +going on between the executioner, the King, and the Queen, who +were all talking at once, while all the rest were quite silent, +and looked very uncomfortable. + + The moment Alice appeared, she was appealed to by all three to +settle the question, and they repeated their arguments to her, +though, as they all spoke at once, she found it very hard indeed +to make out exactly what they said. + + The executioner's argument was, that you couldn't cut off a +head unless there was a body to cut it off from: that he had +never had to do such a thing before, and he wasn't going to begin +at HIS time of life. + + The King's argument was, that anything that had a head could be +beheaded, and that you weren't to talk nonsense. + + The Queen's argument was, that if something wasn't done about +it in less than no time she'd have everybody executed, all round. +(It was this last remark that had made the whole party look so +grave and anxious.) + + Alice could think of nothing else to say but `It belongs to the +Duchess: you'd better ask HER about it.' + + `She's in prison,' the Queen said to the executioner: `fetch +her here.' And the executioner went off like an arrow. + + The Cat's head began fading away the moment he was gone, and, +by the time he had come back with the Dutchess, it had entirely +disappeared; so the King and the executioner ran wildly up and +down looking for it, while the rest of the party went back to the game. + + + + CHAPTER IX + + The Mock Turtle's Story + + + `You can't think how glad I am to see you again, you dear old +thing!' said the Duchess, as she tucked her arm affectionately +into Alice's, and they walked off together. + + Alice was very glad to find her in such a pleasant temper, and +thought to herself that perhaps it was only the pepper that had +made her so savage when they met in the kitchen. + + `When I'M a Duchess,' she said to herself, (not in a very +hopeful tone though), `I won't have any pepper in my kitchen AT +ALL. Soup does very well without--Maybe it's always pepper that +makes people hot-tempered,' she went on, very much pleased at +having found out a new kind of rule, `and vinegar that makes them +sour--and camomile that makes them bitter--and--and barley-sugar +and such things that make children sweet-tempered. I only wish +people knew that: then they wouldn't be so stingy about it, you +know--' + + She had quite forgotten the Duchess by this time, and was a +little startled when she heard her voice close to her ear. +`You're thinking about something, my dear, and that makes you +forget to talk. I can't tell you just now what the moral of that +is, but I shall remember it in a bit.' + + `Perhaps it hasn't one,' Alice ventured to remark. + + `Tut, tut, child!' said the Duchess. `Everything's got a +moral, if only you can find it.' And she squeezed herself up +closer to Alice's side as she spoke. + + Alice did not much like keeping so close to her: first, +because the Duchess was VERY ugly; and secondly, because she was +exactly the right height to rest her chin upon Alice's shoulder, +and it was an uncomfortably sharp chin. However, she did not +like to be rude, so she bore it as well as she could. + + `The game's going on rather better now,' she said, by way of +keeping up the conversation a little. + + `'Tis so,' said the Duchess: `and the moral of that is--"Oh, +'tis love, 'tis love, that makes the world go round!"' + + `Somebody said,' Alice whispered, `that it's done by everybody +minding their own business!' + + `Ah, well! It means much the same thing,' said the Duchess, +digging her sharp little chin into Alice's shoulder as she added, +`and the moral of THAT is--"Take care of the sense, and the +sounds will take care of themselves."' + + `How fond she is of finding morals in things!' Alice thought to +herself. + + `I dare say you're wondering why I don't put my arm round your +waist,' the Duchess said after a pause: `the reason is, that I'm +doubtful about the temper of your flamingo. Shall I try the +experiment?' + + `HE might bite,' Alice cautiously replied, not feeling at all +anxious to have the experiment tried. + + `Very true,' said the Duchess: `flamingoes and mustard both +bite. And the moral of that is--"Birds of a feather flock +together."' + + `Only mustard isn't a bird,' Alice remarked. + + `Right, as usual,' said the Duchess: `what a clear way you +have of putting things!' + + `It's a mineral, I THINK,' said Alice. + + `Of course it is,' said the Duchess, who seemed ready to agree +to everything that Alice said; `there's a large mustard-mine near +here. And the moral of that is--"The more there is of mine, the +less there is of yours."' + + `Oh, I know!' exclaimed Alice, who had not attended to this +last remark, `it's a vegetable. It doesn't look like one, but it +is.' + + `I quite agree with you,' said the Duchess; `and the moral of +that is--"Be what you would seem to be"--or if you'd like it put +more simply--"Never imagine yourself not to be otherwise than +what it might appear to others that what you were or might have +been was not otherwise than what you had been would have appeared +to them to be otherwise."' + + `I think I should understand that better,' Alice said very +politely, `if I had it written down: but I can't quite follow it +as you say it.' + + `That's nothing to what I could say if I chose,' the Duchess +replied, in a pleased tone. + + `Pray don't trouble yourself to say it any longer than that,' +said Alice. + + `Oh, don't talk about trouble!' said the Duchess. `I make you +a present of everything I've said as yet.' + + `A cheap sort of present!' thought Alice. `I'm glad they don't +give birthday presents like that!' But she did not venture to +say it out loud. + + `Thinking again?' the Duchess asked, with another dig of her +sharp little chin. + + `I've a right to think,' said Alice sharply, for she was +beginning to feel a little worried. + + `Just about as much right,' said the Duchess, `as pigs have to +fly; and the m--' + + But here, to Alice's great surprise, the Duchess's voice died +away, even in the middle of her favourite word `moral,' and the +arm that was linked into hers began to tremble. Alice looked up, +and there stood the Queen in front of them, with her arms folded, +frowning like a thunderstorm. + + `A fine day, your Majesty!' the Duchess began in a low, weak +voice. + + `Now, I give you fair warning,' shouted the Queen, stamping on +the ground as she spoke; `either you or your head must be off, +and that in about half no time! Take your choice!' + + The Duchess took her choice, and was gone in a moment. + + `Let's go on with the game,' the Queen said to Alice; and Alice +was too much frightened to say a word, but slowly followed her +back to the croquet-ground. + + The other guests had taken advantage of the Queen's absence, +and were resting in the shade: however, the moment they saw her, +they hurried back to the game, the Queen merely remarking that a +moment's delay would cost them their lives. + + All the time they were playing the Queen never left off +quarrelling with the other players, and shouting `Off with his +head!' or `Off with her head!' Those whom she sentenced were +taken into custody by the soldiers, who of course had to leave +off being arches to do this, so that by the end of half an hour +or so there were no arches left, and all the players, except the +King, the Queen, and Alice, were in custody and under sentence of +execution. + + Then the Queen left off, quite out of breath, and said to +Alice, `Have you seen the Mock Turtle yet?' + + `No,' said Alice. `I don't even know what a Mock Turtle is.' + + `It's the thing Mock Turtle Soup is made from,' said the Queen. + + `I never saw one, or heard of one,' said Alice. + + `Come on, then,' said the Queen, `and he shall tell you his +history,' + + As they walked off together, Alice heard the King say in a low +voice, to the company generally, `You are all pardoned.' `Come, +THAT'S a good thing!' she said to herself, for she had felt quite +unhappy at the number of executions the Queen had ordered. + + They very soon came upon a Gryphon, lying fast asleep in the +sun. (IF you don't know what a Gryphon is, look at the picture.) +`Up, lazy thing!' said the Queen, `and take this young lady to +see the Mock Turtle, and to hear his history. I must go back and +see after some executions I have ordered'; and she walked off, +leaving Alice alone with the Gryphon. Alice did not quite like +the look of the creature, but on the whole she thought it would +be quite as safe to stay with it as to go after that savage +Queen: so she waited. + + The Gryphon sat up and rubbed its eyes: then it watched the +Queen till she was out of sight: then it chuckled. `What fun!' +said the Gryphon, half to itself, half to Alice. + + `What IS the fun?' said Alice. + + `Why, SHE,' said the Gryphon. `It's all her fancy, that: they +never executes nobody, you know. Come on!' + + `Everybody says "come on!" here,' thought Alice, as she went +slowly after it: `I never was so ordered about in all my life, +never!' + + They had not gone far before they saw the Mock Turtle in the +distance, sitting sad and lonely on a little ledge of rock, and, +as they came nearer, Alice could hear him sighing as if his heart +would break. She pitied him deeply. `What is his sorrow?' she +asked the Gryphon, and the Gryphon answered, very nearly in the +same words as before, `It's all his fancy, that: he hasn't got +no sorrow, you know. Come on!' + + So they went up to the Mock Turtle, who looked at them with +large eyes full of tears, but said nothing. + + `This here young lady,' said the Gryphon, `she wants for to +know your history, she do.' + + `I'll tell it her,' said the Mock Turtle in a deep, hollow +tone: `sit down, both of you, and don't speak a word till I've +finished.' + + So they sat down, and nobody spoke for some minutes. Alice +thought to herself, `I don't see how he can EVEN finish, if he +doesn't begin.' But she waited patiently. + + `Once,' said the Mock Turtle at last, with a deep sigh, `I was +a real Turtle.' + + These words were followed by a very long silence, broken only +by an occasional exclamation of `Hjckrrh!' from the Gryphon, and +the constant heavy sobbing of the Mock Turtle. Alice was very +nearly getting up and saying, `Thank you, sir, for your +interesting story,' but she could not help thinking there MUST be +more to come, so she sat still and said nothing. + + `When we were little,' the Mock Turtle went on at last, more +calmly, though still sobbing a little now and then, `we went to +school in the sea. The master was an old Turtle--we used to call +him Tortoise--' + + `Why did you call him Tortoise, if he wasn't one?' Alice asked. + + `We called him Tortoise because he taught us,' said the Mock +Turtle angrily: `really you are very dull!' + + `You ought to be ashamed of yourself for asking such a simple +question,' added the Gryphon; and then they both sat silent and +looked at poor Alice, who felt ready to sink into the earth. At +last the Gryphon said to the Mock Turtle, `Drive on, old fellow! +Don't be all day about it!' and he went on in these words: + + `Yes, we went to school in the sea, though you mayn't believe +it--' + + `I never said I didn't!' interrupted Alice. + + `You did,' said the Mock Turtle. + + `Hold your tongue!' added the Gryphon, before Alice could speak +again. The Mock Turtle went on. + + `We had the best of educations--in fact, we went to school +every day--' + + `I'VE been to a day-school, too,' said Alice; `you needn't be +so proud as all that.' + + `With extras?' asked the Mock Turtle a little anxiously. + + `Yes,' said Alice, `we learned French and music.' + + `And washing?' said the Mock Turtle. + + `Certainly not!' said Alice indignantly. + + `Ah! then yours wasn't a really good school,' said the Mock +Turtle in a tone of great relief. `Now at OURS they had at the +end of the bill, "French, music, AND WASHING--extra."' + + `You couldn't have wanted it much,' said Alice; `living at the +bottom of the sea.' + + `I couldn't afford to learn it.' said the Mock Turtle with a +sigh. `I only took the regular course.' + + `What was that?' inquired Alice. + + `Reeling and Writhing, of course, to begin with,' the Mock +Turtle replied; `and then the different branches of Arithmetic-- +Ambition, Distraction, Uglification, and Derision.' + + `I never heard of "Uglification,"' Alice ventured to say. `What +is it?' + + The Gryphon lifted up both its paws in surprise. `What! Never +heard of uglifying!' it exclaimed. `You know what to beautify +is, I suppose?' + + `Yes,' said Alice doubtfully: `it means--to--make--anything-- +prettier.' + + `Well, then,' the Gryphon went on, `if you don't know what to +uglify is, you ARE a simpleton.' + + Alice did not feel encouraged to ask any more questions about +it, so she turned to the Mock Turtle, and said `What else had you +to learn?' + + `Well, there was Mystery,' the Mock Turtle replied, counting +off the subjects on his flappers, `--Mystery, ancient and modern, +with Seaography: then Drawling--the Drawling-master was an old +conger-eel, that used to come once a week: HE taught us +Drawling, Stretching, and Fainting in Coils.' + + `What was THAT like?' said Alice. + + `Well, I can't show it you myself,' the Mock Turtle said: `I'm +too stiff. And the Gryphon never learnt it.' + + `Hadn't time,' said the Gryphon: `I went to the Classics +master, though. He was an old crab, HE was.' + + `I never went to him,' the Mock Turtle said with a sigh: `he +taught Laughing and Grief, they used to say.' + + `So he did, so he did,' said the Gryphon, sighing in his turn; +and both creatures hid their faces in their paws. + + `And how many hours a day did you do lessons?' said Alice, in a +hurry to change the subject. + + `Ten hours the first day,' said the Mock Turtle: `nine the +next, and so on.' + + `What a curious plan!' exclaimed Alice. + + `That's the reason they're called lessons,' the Gryphon +remarked: `because they lessen from day to day.' + + This was quite a new idea to Alice, and she thought it over a +little before she made her next remark. `Then the eleventh day +must have been a holiday?' + + `Of course it was,' said the Mock Turtle. + + `And how did you manage on the twelfth?' Alice went on eagerly. + + `That's enough about lessons,' the Gryphon interrupted in a +very decided tone: `tell her something about the games now.' + + + + CHAPTER X + + The Lobster Quadrille + + + The Mock Turtle sighed deeply, and drew the back of one flapper +across his eyes. He looked at Alice, and tried to speak, but for +a minute or two sobs choked his voice. `Same as if he had a bone +in his throat,' said the Gryphon: and it set to work shaking him +and punching him in the back. At last the Mock Turtle recovered +his voice, and, with tears running down his cheeks, he went on +again:-- + + `You may not have lived much under the sea--' (`I haven't,' +said Alice)--`and perhaps you were never even introduced to a lobster--' +(Alice began to say `I once tasted--' but checked herself hastily, +and said `No, never') `--so you can have no idea what a delightful +thing a Lobster Quadrille is!' + + `No, indeed,' said Alice. `What sort of a dance is it?' + + `Why,' said the Gryphon, `you first form into a line along the +sea-shore--' + + `Two lines!' cried the Mock Turtle. `Seals, turtles, salmon, +and so on; then, when you've cleared all the jelly-fish out of +the way--' + + `THAT generally takes some time,' interrupted the Gryphon. + + `--you advance twice--' + + `Each with a lobster as a partner!' cried the Gryphon. + + `Of course,' the Mock Turtle said: `advance twice, set to +partners--' + + `--change lobsters, and retire in same order,' continued the +Gryphon. + + `Then, you know,' the Mock Turtle went on, `you throw the--' + + `The lobsters!' shouted the Gryphon, with a bound into the air. + + `--as far out to sea as you can--' + + `Swim after them!' screamed the Gryphon. + + `Turn a somersault in the sea!' cried the Mock Turtle, +capering wildly about. + + `Back to land again, and that's all the first figure,' said the +Mock Turtle, suddenly dropping his voice; and the two creatures, +who had been jumping about like mad things all this time, sat +down again very sadly and quietly, and looked at Alice. + + `It must be a very pretty dance,' said Alice timidly. + + `Would you like to see a little of it?' said the Mock Turtle. + + `Very much indeed,' said Alice. + + `Come, let's try the first figure!' said the Mock Turtle to the +Gryphon. `We can do without lobsters, you know. Which shall +sing?' + + `Oh, YOU sing,' said the Gryphon. `I've forgotten the words.' + + So they began solemnly dancing round and round Alice, every now +and then treading on her toes when they passed too close, and +waving their forepaws to mark the time, while the Mock Turtle +sang this, very slowly and sadly:-- + + +`"Will you walk a little faster?" said a whiting to a snail. +"There's a porpoise close behind us, and he's treading on my + tail. +See how eagerly the lobsters and the turtles all advance! +They are waiting on the shingle--will you come and join the +dance? + +Will you, won't you, will you, won't you, will you join the +dance? +Will you, won't you, will you, won't you, won't you join the +dance? + + +"You can really have no notion how delightful it will be +When they take us up and throw us, with the lobsters, out to + sea!" +But the snail replied "Too far, too far!" and gave a look + askance-- +Said he thanked the whiting kindly, but he would not join the + dance. + Would not, could not, would not, could not, would not join + the dance. + Would not, could not, would not, could not, could not join + the dance. + +`"What matters it how far we go?" his scaly friend replied. +"There is another shore, you know, upon the other side. +The further off from England the nearer is to France-- +Then turn not pale, beloved snail, but come and join the dance. + + Will you, won't you, will you, won't you, will you join the + dance? + Will you, won't you, will you, won't you, won't you join the + dance?"' + + + + `Thank you, it's a very interesting dance to watch,' said +Alice, feeling very glad that it was over at last: `and I do so +like that curious song about the whiting!' + + `Oh, as to the whiting,' said the Mock Turtle, `they--you've +seen them, of course?' + + `Yes,' said Alice, `I've often seen them at dinn--' she +checked herself hastily. + + `I don't know where Dinn may be,' said the Mock Turtle, `but +if you've seen them so often, of course you know what they're +like.' + + `I believe so,' Alice replied thoughtfully. `They have their +tails in their mouths--and they're all over crumbs.' + + `You're wrong about the crumbs,' said the Mock Turtle: +`crumbs would all wash off in the sea. But they HAVE their tails +in their mouths; and the reason is--' here the Mock Turtle +yawned and shut his eyes.--`Tell her about the reason and all +that,' he said to the Gryphon. + + `The reason is,' said the Gryphon, `that they WOULD go with +the lobsters to the dance. So they got thrown out to sea. So +they had to fall a long way. So they got their tails fast in +their mouths. So they couldn't get them out again. That's all.' + + `Thank you,' said Alice, `it's very interesting. I never knew +so much about a whiting before.' + + `I can tell you more than that, if you like,' said the +Gryphon. `Do you know why it's called a whiting?' + + `I never thought about it,' said Alice. `Why?' + + `IT DOES THE BOOTS AND SHOES.' the Gryphon replied very +solemnly. + + Alice was thoroughly puzzled. `Does the boots and shoes!' she +repeated in a wondering tone. + + `Why, what are YOUR shoes done with?' said the Gryphon. `I +mean, what makes them so shiny?' + + Alice looked down at them, and considered a little before she +gave her answer. `They're done with blacking, I believe.' + + `Boots and shoes under the sea,' the Gryphon went on in a deep +voice, `are done with a whiting. Now you know.' + + `And what are they made of?' Alice asked in a tone of great +curiosity. + + `Soles and eels, of course,' the Gryphon replied rather +impatiently: `any shrimp could have told you that.' + + `If I'd been the whiting,' said Alice, whose thoughts were +still running on the song, `I'd have said to the porpoise, "Keep +back, please: we don't want YOU with us!"' + + `They were obliged to have him with them,' the Mock Turtle +said: `no wise fish would go anywhere without a porpoise.' + + `Wouldn't it really?' said Alice in a tone of great surprise. + + `Of course not,' said the Mock Turtle: `why, if a fish came +to ME, and told me he was going a journey, I should say "With +what porpoise?"' + + `Don't you mean "purpose"?' said Alice. + + `I mean what I say,' the Mock Turtle replied in an offended +tone. And the Gryphon added `Come, let's hear some of YOUR +adventures.' + + `I could tell you my adventures--beginning from this morning,' +said Alice a little timidly: `but it's no use going back to +yesterday, because I was a different person then.' + + `Explain all that,' said the Mock Turtle. + + `No, no! The adventures first,' said the Gryphon in an +impatient tone: `explanations take such a dreadful time.' + + So Alice began telling them her adventures from the time when +she first saw the White Rabbit. She was a little nervous about +it just at first, the two creatures got so close to her, one on +each side, and opened their eyes and mouths so VERY wide, but she +gained courage as she went on. Her listeners were perfectly +quiet till she got to the part about her repeating `YOU ARE OLD, +FATHER WILLIAM,' to the Caterpillar, and the words all coming +different, and then the Mock Turtle drew a long breath, and said +`That's very curious.' + + `It's all about as curious as it can be,' said the Gryphon. + + `It all came different!' the Mock Turtle repeated +thoughtfully. `I should like to hear her try and repeat +something now. Tell her to begin.' He looked at the Gryphon as +if he thought it had some kind of authority over Alice. + + `Stand up and repeat "'TIS THE VOICE OF THE SLUGGARD,"' said +the Gryphon. + + `How the creatures order one about, and make one repeat +lessons!' thought Alice; `I might as well be at school at once.' +However, she got up, and began to repeat it, but her head was so +full of the Lobster Quadrille, that she hardly knew what she was +saying, and the words came very queer indeed:-- + + `'Tis the voice of the Lobster; I heard him declare, + "You have baked me too brown, I must sugar my hair." + As a duck with its eyelids, so he with his nose + Trims his belt and his buttons, and turns out his toes.' + + [later editions continued as follows + When the sands are all dry, he is gay as a lark, + And will talk in contemptuous tones of the Shark, + But, when the tide rises and sharks are around, + His voice has a timid and tremulous sound.] + + `That's different from what I used to say when I was a child,' +said the Gryphon. + + `Well, I never heard it before,' said the Mock Turtle; `but it +sounds uncommon nonsense.' + + Alice said nothing; she had sat down with her face in her +hands, wondering if anything would EVER happen in a natural way +again. + + `I should like to have it explained,' said the Mock Turtle. + + `She can't explain it,' said the Gryphon hastily. `Go on with +the next verse.' + + `But about his toes?' the Mock Turtle persisted. `How COULD +he turn them out with his nose, you know?' + + `It's the first position in dancing.' Alice said; but was +dreadfully puzzled by the whole thing, and longed to change the +subject. + + `Go on with the next verse,' the Gryphon repeated impatiently: +`it begins "I passed by his garden."' + + Alice did not dare to disobey, though she felt sure it would +all come wrong, and she went on in a trembling voice:-- + + `I passed by his garden, and marked, with one eye, + How the Owl and the Panther were sharing a pie--' + + [later editions continued as follows + The Panther took pie-crust, and gravy, and meat, + While the Owl had the dish as its share of the treat. + When the pie was all finished, the Owl, as a boon, + Was kindly permitted to pocket the spoon: + While the Panther received knife and fork with a growl, + And concluded the banquet--] + + `What IS the use of repeating all that stuff,' the Mock Turtle +interrupted, `if you don't explain it as you go on? It's by far +the most confusing thing I ever heard!' + + `Yes, I think you'd better leave off,' said the Gryphon: and +Alice was only too glad to do so. + + `Shall we try another figure of the Lobster Quadrille?' the +Gryphon went on. `Or would you like the Mock Turtle to sing you +a song?' + + `Oh, a song, please, if the Mock Turtle would be so kind,' +Alice replied, so eagerly that the Gryphon said, in a rather +offended tone, `Hm! No accounting for tastes! Sing her "Turtle +Soup," will you, old fellow?' + + The Mock Turtle sighed deeply, and began, in a voice sometimes +choked with sobs, to sing this:-- + + + `Beautiful Soup, so rich and green, + Waiting in a hot tureen! + Who for such dainties would not stoop? + Soup of the evening, beautiful Soup! + Soup of the evening, beautiful Soup! + Beau--ootiful Soo--oop! + Beau--ootiful Soo--oop! + Soo--oop of the e--e--evening, + Beautiful, beautiful Soup! + + `Beautiful Soup! Who cares for fish, + Game, or any other dish? + Who would not give all else for two p + ennyworth only of beautiful Soup? + Pennyworth only of beautiful Soup? + Beau--ootiful Soo--oop! + Beau--ootiful Soo--oop! + Soo--oop of the e--e--evening, + Beautiful, beauti--FUL SOUP!' + + `Chorus again!' cried the Gryphon, and the Mock Turtle had +just begun to repeat it, when a cry of `The trial's beginning!' +was heard in the distance. + + `Come on!' cried the Gryphon, and, taking Alice by the hand, +it hurried off, without waiting for the end of the song. + + `What trial is it?' Alice panted as she ran; but the Gryphon +only answered `Come on!' and ran the faster, while more and more +faintly came, carried on the breeze that followed them, the +melancholy words:-- + + `Soo--oop of the e--e--evening, + Beautiful, beautiful Soup!' + + + + CHAPTER XI + + Who Stole the Tarts? + + + The King and Queen of Hearts were seated on their throne when +they arrived, with a great crowd assembled about them--all sorts +of little birds and beasts, as well as the whole pack of cards: +the Knave was standing before them, in chains, with a soldier on +each side to guard him; and near the King was the White Rabbit, +with a trumpet in one hand, and a scroll of parchment in the +other. In the very middle of the court was a table, with a large +dish of tarts upon it: they looked so good, that it made Alice +quite hungry to look at them--`I wish they'd get the trial done,' +she thought, `and hand round the refreshments!' But there seemed +to be no chance of this, so she began looking at everything about +her, to pass away the time. + + Alice had never been in a court of justice before, but she had +read about them in books, and she was quite pleased to find that +she knew the name of nearly everything there. `That's the +judge,' she said to herself, `because of his great wig.' + + The judge, by the way, was the King; and as he wore his crown +over the wig, (look at the frontispiece if you want to see how he +did it,) he did not look at all comfortable, and it was certainly +not becoming. + + `And that's the jury-box,' thought Alice, `and those twelve +creatures,' (she was obliged to say `creatures,' you see, because +some of them were animals, and some were birds,) `I suppose they +are the jurors.' She said this last word two or three times over +to herself, being rather proud of it: for she thought, and +rightly too, that very few little girls of her age knew the +meaning of it at all. However, `jury-men' would have done just +as well. + + The twelve jurors were all writing very busily on slates. +`What are they doing?' Alice whispered to the Gryphon. `They +can't have anything to put down yet, before the trial's begun.' + + `They're putting down their names,' the Gryphon whispered in +reply, `for fear they should forget them before the end of the +trial.' + + `Stupid things!' Alice began in a loud, indignant voice, but +she stopped hastily, for the White Rabbit cried out, `Silence in +the court!' and the King put on his spectacles and looked +anxiously round, to make out who was talking. + + Alice could see, as well as if she were looking over their +shoulders, that all the jurors were writing down `stupid things!' +on their slates, and she could even make out that one of them +didn't know how to spell `stupid,' and that he had to ask his +neighbour to tell him. `A nice muddle their slates'll be in +before the trial's over!' thought Alice. + + One of the jurors had a pencil that squeaked. This of course, +Alice could not stand, and she went round the court and got +behind him, and very soon found an opportunity of taking it +away. She did it so quickly that the poor little juror (it was +Bill, the Lizard) could not make out at all what had become of +it; so, after hunting all about for it, he was obliged to write +with one finger for the rest of the day; and this was of very +little use, as it left no mark on the slate. + + `Herald, read the accusation!' said the King. + + On this the White Rabbit blew three blasts on the trumpet, and +then unrolled the parchment scroll, and read as follows:-- + + `The Queen of Hearts, she made some tarts, + All on a summer day: + The Knave of Hearts, he stole those tarts, + And took them quite away!' + + `Consider your verdict,' the King said to the jury. + + `Not yet, not yet!' the Rabbit hastily interrupted. `There's +a great deal to come before that!' + + `Call the first witness,' said the King; and the White Rabbit +blew three blasts on the trumpet, and called out, `First +witness!' + + The first witness was the Hatter. He came in with a teacup in +one hand and a piece of bread-and-butter in the other. `I beg +pardon, your Majesty,' he began, `for bringing these in: but I +hadn't quite finished my tea when I was sent for.' + + `You ought to have finished,' said the King. `When did you +begin?' + + The Hatter looked at the March Hare, who had followed him into +the court, arm-in-arm with the Dormouse. `Fourteenth of March, I +think it was,' he said. + + `Fifteenth,' said the March Hare. + + `Sixteenth,' added the Dormouse. + + `Write that down,' the King said to the jury, and the jury +eagerly wrote down all three dates on their slates, and then +added them up, and reduced the answer to shillings and pence. + + `Take off your hat,' the King said to the Hatter. + + `It isn't mine,' said the Hatter. + + `Stolen!' the King exclaimed, turning to the jury, who +instantly made a memorandum of the fact. + + `I keep them to sell,' the Hatter added as an explanation; +`I've none of my own. I'm a hatter.' + + Here the Queen put on her spectacles, and began staring at the +Hatter, who turned pale and fidgeted. + + `Give your evidence,' said the King; `and don't be nervous, or +I'll have you executed on the spot.' + + This did not seem to encourage the witness at all: he kept +shifting from one foot to the other, looking uneasily at the +Queen, and in his confusion he bit a large piece out of his +teacup instead of the bread-and-butter. + + Just at this moment Alice felt a very curious sensation, which +puzzled her a good deal until she made out what it was: she was +beginning to grow larger again, and she thought at first she +would get up and leave the court; but on second thoughts she +decided to remain where she was as long as there was room for +her. + + `I wish you wouldn't squeeze so.' said the Dormouse, who was +sitting next to her. `I can hardly breathe.' + + `I can't help it,' said Alice very meekly: `I'm growing.' + + `You've no right to grow here,' said the Dormouse. + + `Don't talk nonsense,' said Alice more boldly: `you know +you're growing too.' + + `Yes, but I grow at a reasonable pace,' said the Dormouse: +`not in that ridiculous fashion.' And he got up very sulkily +and crossed over to the other side of the court. + + All this time the Queen had never left off staring at the +Hatter, and, just as the Dormouse crossed the court, she said to +one of the officers of the court, `Bring me the list of the +singers in the last concert!' on which the wretched Hatter +trembled so, that he shook both his shoes off. + + `Give your evidence,' the King repeated angrily, `or I'll have +you executed, whether you're nervous or not.' + + `I'm a poor man, your Majesty,' the Hatter began, in a +trembling voice, `--and I hadn't begun my tea--not above a week +or so--and what with the bread-and-butter getting so thin--and +the twinkling of the tea--' + + `The twinkling of the what?' said the King. + + `It began with the tea,' the Hatter replied. + + `Of course twinkling begins with a T!' said the King sharply. +`Do you take me for a dunce? Go on!' + + `I'm a poor man,' the Hatter went on, `and most things +twinkled after that--only the March Hare said--' + + `I didn't!' the March Hare interrupted in a great hurry. + + `You did!' said the Hatter. + + `I deny it!' said the March Hare. + + `He denies it,' said the King: `leave out that part.' + + `Well, at any rate, the Dormouse said--' the Hatter went on, +looking anxiously round to see if he would deny it too: but the +Dormouse denied nothing, being fast asleep. + + `After that,' continued the Hatter, `I cut some more bread- +and-butter--' + + `But what did the Dormouse say?' one of the jury asked. + + `That I can't remember,' said the Hatter. + + `You MUST remember,' remarked the King, `or I'll have you +executed.' + + The miserable Hatter dropped his teacup and bread-and-butter, +and went down on one knee. `I'm a poor man, your Majesty,' he +began. + + `You're a very poor speaker,' said the King. + + Here one of the guinea-pigs cheered, and was immediately +suppressed by the officers of the court. (As that is rather a +hard word, I will just explain to you how it was done. They had +a large canvas bag, which tied up at the mouth with strings: +into this they slipped the guinea-pig, head first, and then sat +upon it.) + + `I'm glad I've seen that done,' thought Alice. `I've so often +read in the newspapers, at the end of trials, "There was some +attempts at applause, which was immediately suppressed by the +officers of the court," and I never understood what it meant +till now.' + + `If that's all you know about it, you may stand down,' +continued the King. + + `I can't go no lower,' said the Hatter: `I'm on the floor, as +it is.' + + `Then you may SIT down,' the King replied. + + Here the other guinea-pig cheered, and was suppressed. + + `Come, that finished the guinea-pigs!' thought Alice. `Now we +shall get on better.' + + `I'd rather finish my tea,' said the Hatter, with an anxious +look at the Queen, who was reading the list of singers. + + `You may go,' said the King, and the Hatter hurriedly left the +court, without even waiting to put his shoes on. + + `--and just take his head off outside,' the Queen added to one +of the officers: but the Hatter was out of sight before the +officer could get to the door. + + `Call the next witness!' said the King. + + The next witness was the Duchess's cook. She carried the +pepper-box in her hand, and Alice guessed who it was, even before +she got into the court, by the way the people near the door began +sneezing all at once. + + `Give your evidence,' said the King. + + `Shan't,' said the cook. + + The King looked anxiously at the White Rabbit, who said in a +low voice, `Your Majesty must cross-examine THIS witness.' + + `Well, if I must, I must,' the King said, with a melancholy +air, and, after folding his arms and frowning at the cook till +his eyes were nearly out of sight, he said in a deep voice, `What +are tarts made of?' + + `Pepper, mostly,' said the cook. + + `Treacle,' said a sleepy voice behind her. + + `Collar that Dormouse,' the Queen shrieked out. `Behead that +Dormouse! Turn that Dormouse out of court! Suppress him! Pinch +him! Off with his whiskers!' + + For some minutes the whole court was in confusion, getting the +Dormouse turned out, and, by the time they had settled down +again, the cook had disappeared. + + `Never mind!' said the King, with an air of great relief. +`Call the next witness.' And he added in an undertone to the +Queen, `Really, my dear, YOU must cross-examine the next witness. +It quite makes my forehead ache!' + + Alice watched the White Rabbit as he fumbled over the list, +feeling very curious to see what the next witness would be like, +`--for they haven't got much evidence YET,' she said to herself. +Imagine her surprise, when the White Rabbit read out, at the top +of his shrill little voice, the name `Alice!' + + + + CHAPTER XII + + Alice's Evidence + + + `Here!' cried Alice, quite forgetting in the flurry of the +moment how large she had grown in the last few minutes, and she +jumped up in such a hurry that she tipped over the jury-box with +the edge of her skirt, upsetting all the jurymen on to the heads +of the crowd below, and there they lay sprawling about, reminding +her very much of a globe of goldfish she had accidentally upset +the week before. + + `Oh, I BEG your pardon!' she exclaimed in a tone of great +dismay, and began picking them up again as quickly as she could, +for the accident of the goldfish kept running in her head, and +she had a vague sort of idea that they must be collected at once +and put back into the jury-box, or they would die. + + `The trial cannot proceed,' said the King in a very grave +voice, `until all the jurymen are back in their proper places-- +ALL,' he repeated with great emphasis, looking hard at Alice as +he said do. + + Alice looked at the jury-box, and saw that, in her haste, she +had put the Lizard in head downwards, and the poor little thing +was waving its tail about in a melancholy way, being quite unable +to move. She soon got it out again, and put it right; `not that +it signifies much,' she said to herself; `I should think it +would be QUITE as much use in the trial one way up as the other.' + + As soon as the jury had a little recovered from the shock of +being upset, and their slates and pencils had been found and +handed back to them, they set to work very diligently to write +out a history of the accident, all except the Lizard, who seemed +too much overcome to do anything but sit with its mouth open, +gazing up into the roof of the court. + + `What do you know about this business?' the King said to +Alice. + + `Nothing,' said Alice. + + `Nothing WHATEVER?' persisted the King. + + `Nothing whatever,' said Alice. + + `That's very important,' the King said, turning to the jury. +They were just beginning to write this down on their slates, when +the White Rabbit interrupted: `UNimportant, your Majesty means, +of course,' he said in a very respectful tone, but frowning and +making faces at him as he spoke. + + `UNimportant, of course, I meant,' the King hastily said, and +went on to himself in an undertone, `important--unimportant-- +unimportant--important--' as if he were trying which word +sounded best. + + Some of the jury wrote it down `important,' and some +`unimportant.' Alice could see this, as she was near enough to +look over their slates; `but it doesn't matter a bit,' she +thought to herself. + + At this moment the King, who had been for some time busily +writing in his note-book, cackled out `Silence!' and read out +from his book, `Rule Forty-two. ALL PERSONS MORE THAN A MILE +HIGH TO LEAVE THE COURT.' + + Everybody looked at Alice. + + `I'M not a mile high,' said Alice. + + `You are,' said the King. + + `Nearly two miles high,' added the Queen. + + `Well, I shan't go, at any rate,' said Alice: `besides, +that's not a regular rule: you invented it just now.' + + `It's the oldest rule in the book,' said the King. + + `Then it ought to be Number One,' said Alice. + + The King turned pale, and shut his note-book hastily. +`Consider your verdict,' he said to the jury, in a low, trembling +voice. + + `There's more evidence to come yet, please your Majesty,' said +the White Rabbit, jumping up in a great hurry; `this paper has +just been picked up.' + + `What's in it?' said the Queen. + + `I haven't opened it yet,' said the White Rabbit, `but it seems +to be a letter, written by the prisoner to--to somebody.' + + `It must have been that,' said the King, `unless it was +written to nobody, which isn't usual, you know.' + + `Who is it directed to?' said one of the jurymen. + + `It isn't directed at all,' said the White Rabbit; `in fact, +there's nothing written on the OUTSIDE.' He unfolded the paper +as he spoke, and added `It isn't a letter, after all: it's a set +of verses.' + + `Are they in the prisoner's handwriting?' asked another of +they jurymen. + + `No, they're not,' said the White Rabbit, `and that's the +queerest thing about it.' (The jury all looked puzzled.) + + `He must have imitated somebody else's hand,' said the King. +(The jury all brightened up again.) + + `Please your Majesty,' said the Knave, `I didn't write it, and +they can't prove I did: there's no name signed at the end.' + + `If you didn't sign it,' said the King, `that only makes the +matter worse. You MUST have meant some mischief, or else you'd +have signed your name like an honest man.' + + There was a general clapping of hands at this: it was the +first really clever thing the King had said that day. + + `That PROVES his guilt,' said the Queen. + + `It proves nothing of the sort!' said Alice. `Why, you don't +even know what they're about!' + + `Read them,' said the King. + + The White Rabbit put on his spectacles. `Where shall I begin, +please your Majesty?' he asked. + + `Begin at the beginning,' the King said gravely, `and go on +till you come to the end: then stop.' + + These were the verses the White Rabbit read:-- + + `They told me you had been to her, + And mentioned me to him: + She gave me a good character, + But said I could not swim. + + He sent them word I had not gone + (We know it to be true): + If she should push the matter on, + What would become of you? + + I gave her one, they gave him two, + You gave us three or more; + They all returned from him to you, + Though they were mine before. + + If I or she should chance to be + Involved in this affair, + He trusts to you to set them free, + Exactly as we were. + + My notion was that you had been + (Before she had this fit) + An obstacle that came between + Him, and ourselves, and it. + + Don't let him know she liked them best, + For this must ever be + A secret, kept from all the rest, + Between yourself and me.' + + `That's the most important piece of evidence we've heard yet,' +said the King, rubbing his hands; `so now let the jury--' + + `If any one of them can explain it,' said Alice, (she had +grown so large in the last few minutes that she wasn't a bit +afraid of interrupting him,) `I'll give him sixpence. _I_ don't +believe there's an atom of meaning in it.' + + The jury all wrote down on their slates, `SHE doesn't believe +there's an atom of meaning in it,' but none of them attempted to +explain the paper. + + `If there's no meaning in it,' said the King, `that saves a +world of trouble, you know, as we needn't try to find any. And +yet I don't know,' he went on, spreading out the verses on his +knee, and looking at them with one eye; `I seem to see some +meaning in them, after all. "--SAID I COULD NOT SWIM--" you +can't swim, can you?' he added, turning to the Knave. + + The Knave shook his head sadly. `Do I look like it?' he said. +(Which he certainly did NOT, being made entirely of cardboard.) + + `All right, so far,' said the King, and he went on muttering +over the verses to himself: `"WE KNOW IT TO BE TRUE--" that's +the jury, of course-- "I GAVE HER ONE, THEY GAVE HIM TWO--" why, +that must be what he did with the tarts, you know--' + + `But, it goes on "THEY ALL RETURNED FROM HIM TO YOU,"' said +Alice. + + `Why, there they are!' said the King triumphantly, pointing to +the tarts on the table. `Nothing can be clearer than THAT. +Then again--"BEFORE SHE HAD THIS FIT--" you never had fits, my +dear, I think?' he said to the Queen. + + `Never!' said the Queen furiously, throwing an inkstand at the +Lizard as she spoke. (The unfortunate little Bill had left off +writing on his slate with one finger, as he found it made no +mark; but he now hastily began again, using the ink, that was +trickling down his face, as long as it lasted.) + + `Then the words don't FIT you,' said the King, looking round +the court with a smile. There was a dead silence. + + `It's a pun!' the King added in an offended tone, and +everybody laughed, `Let the jury consider their verdict,' the +King said, for about the twentieth time that day. + + `No, no!' said the Queen. `Sentence first--verdict afterwards.' + + `Stuff and nonsense!' said Alice loudly. `The idea of having +the sentence first!' + + `Hold your tongue!' said the Queen, turning purple. + + `I won't!' said Alice. + + `Off with her head!' the Queen shouted at the top of her voice. +Nobody moved. + + `Who cares for you?' said Alice, (she had grown to her full +size by this time.) `You're nothing but a pack of cards!' + + At this the whole pack rose up into the air, and came flying +down upon her: she gave a little scream, half of fright and half +of anger, and tried to beat them off, and found herself lying on +the bank, with her head in the lap of her sister, who was gently +brushing away some dead leaves that had fluttered down from the +trees upon her face. + + `Wake up, Alice dear!' said her sister; `Why, what a long +sleep you've had!' + + `Oh, I've had such a curious dream!' said Alice, and she told +her sister, as well as she could remember them, all these strange +Adventures of hers that you have just been reading about; and +when she had finished, her sister kissed her, and said, `It WAS a +curious dream, dear, certainly: but now run in to your tea; it's +getting late.' So Alice got up and ran off, thinking while she +ran, as well she might, what a wonderful dream it had been. + + But her sister sat still just as she left her, leaning her +head on her hand, watching the setting sun, and thinking of +little Alice and all her wonderful Adventures, till she too began +dreaming after a fashion, and this was her dream:-- + + First, she dreamed of little Alice herself, and once again the +tiny hands were clasped upon her knee, and the bright eager eyes +were looking up into hers--she could hear the very tones of her +voice, and see that queer little toss of her head to keep back +the wandering hair that WOULD always get into her eyes--and +still as she listened, or seemed to listen, the whole place +around her became alive the strange creatures of her little +sister's dream. + + The long grass rustled at her feet as the White Rabbit hurried +by--the frightened Mouse splashed his way through the +neighbouring pool--she could hear the rattle of the teacups as +the March Hare and his friends shared their never-ending meal, +and the shrill voice of the Queen ordering off her unfortunate +guests to execution--once more the pig-baby was sneezing on the +Duchess's knee, while plates and dishes crashed around it--once +more the shriek of the Gryphon, the squeaking of the Lizard's +slate-pencil, and the choking of the suppressed guinea-pigs, +filled the air, mixed up with the distant sobs of the miserable +Mock Turtle. + + So she sat on, with closed eyes, and half believed herself in +Wonderland, though she knew she had but to open them again, and +all would change to dull reality--the grass would be only +rustling in the wind, and the pool rippling to the waving of the +reeds--the rattling teacups would change to tinkling sheep- +bells, and the Queen's shrill cries to the voice of the shepherd +boy--and the sneeze of the baby, the shriek of the Gryphon, and +all thy other queer noises, would change (she knew) to the +confused clamour of the busy farm-yard--while the lowing of the +cattle in the distance would take the place of the Mock Turtle's +heavy sobs. + + Lastly, she pictured to herself how this same little sister of +hers would, in the after-time, be herself a grown woman; and how +she would keep, through all her riper years, the simple and +loving heart of her childhood: and how she would gather about +her other little children, and make THEIR eyes bright and eager +with many a strange tale, perhaps even with the dream of +Wonderland of long ago: and how she would feel with all their +simple sorrows, and find a pleasure in all their simple joys, +remembering her own child-life, and the happy summer days. + + THE END + \ No newline at end of file diff --git a/target/snappy-1.1.10/testdata/asyoulik.txt b/target/snappy-1.1.10/testdata/asyoulik.txt new file mode 100644 index 00000000..88dc7b60 --- /dev/null +++ b/target/snappy-1.1.10/testdata/asyoulik.txt @@ -0,0 +1,4122 @@ + AS YOU LIKE IT + + + DRAMATIS PERSONAE + + +DUKE SENIOR living in banishment. + +DUKE FREDERICK his brother, an usurper of his dominions. + + +AMIENS | + | lords attending on the banished duke. +JAQUES | + + +LE BEAU a courtier attending upon Frederick. + +CHARLES wrestler to Frederick. + + +OLIVER | + | +JAQUES (JAQUES DE BOYS:) | sons of Sir Rowland de Boys. + | +ORLANDO | + + +ADAM | + | servants to Oliver. +DENNIS | + + +TOUCHSTONE a clown. + +SIR OLIVER MARTEXT a vicar. + + +CORIN | + | shepherds. +SILVIUS | + + +WILLIAM a country fellow in love with Audrey. + + A person representing HYMEN. (HYMEN:) + +ROSALIND daughter to the banished duke. + +CELIA daughter to Frederick. + +PHEBE a shepherdess. + +AUDREY a country wench. + + Lords, pages, and attendants, &c. + (Forester:) + (A Lord:) + (First Lord:) + (Second Lord:) + (First Page:) + (Second Page:) + + +SCENE Oliver's house; Duke Frederick's court; and the + Forest of Arden. + + + + + AS YOU LIKE IT + + +ACT I + + + +SCENE I Orchard of Oliver's house. + + + [Enter ORLANDO and ADAM] + +ORLANDO As I remember, Adam, it was upon this fashion + bequeathed me by will but poor a thousand crowns, + and, as thou sayest, charged my brother, on his + blessing, to breed me well: and there begins my + sadness. My brother Jaques he keeps at school, and + report speaks goldenly of his profit: for my part, + he keeps me rustically at home, or, to speak more + properly, stays me here at home unkept; for call you + that keeping for a gentleman of my birth, that + differs not from the stalling of an ox? His horses + are bred better; for, besides that they are fair + with their feeding, they are taught their manage, + and to that end riders dearly hired: but I, his + brother, gain nothing under him but growth; for the + which his animals on his dunghills are as much + bound to him as I. Besides this nothing that he so + plentifully gives me, the something that nature gave + me his countenance seems to take from me: he lets + me feed with his hinds, bars me the place of a + brother, and, as much as in him lies, mines my + gentility with my education. This is it, Adam, that + grieves me; and the spirit of my father, which I + think is within me, begins to mutiny against this + servitude: I will no longer endure it, though yet I + know no wise remedy how to avoid it. + +ADAM Yonder comes my master, your brother. + +ORLANDO Go apart, Adam, and thou shalt hear how he will + shake me up. + + [Enter OLIVER] + +OLIVER Now, sir! what make you here? + +ORLANDO Nothing: I am not taught to make any thing. + +OLIVER What mar you then, sir? + +ORLANDO Marry, sir, I am helping you to mar that which God + made, a poor unworthy brother of yours, with idleness. + +OLIVER Marry, sir, be better employed, and be naught awhile. + +ORLANDO Shall I keep your hogs and eat husks with them? + What prodigal portion have I spent, that I should + come to such penury? + +OLIVER Know you where your are, sir? + +ORLANDO O, sir, very well; here in your orchard. + +OLIVER Know you before whom, sir? + +ORLANDO Ay, better than him I am before knows me. I know + you are my eldest brother; and, in the gentle + condition of blood, you should so know me. The + courtesy of nations allows you my better, in that + you are the first-born; but the same tradition + takes not away my blood, were there twenty brothers + betwixt us: I have as much of my father in me as + you; albeit, I confess, your coming before me is + nearer to his reverence. + +OLIVER What, boy! + +ORLANDO Come, come, elder brother, you are too young in this. + +OLIVER Wilt thou lay hands on me, villain? + +ORLANDO I am no villain; I am the youngest son of Sir + Rowland de Boys; he was my father, and he is thrice + a villain that says such a father begot villains. + Wert thou not my brother, I would not take this hand + from thy throat till this other had pulled out thy + tongue for saying so: thou hast railed on thyself. + +ADAM Sweet masters, be patient: for your father's + remembrance, be at accord. + +OLIVER Let me go, I say. + +ORLANDO I will not, till I please: you shall hear me. My + father charged you in his will to give me good + education: you have trained me like a peasant, + obscuring and hiding from me all gentleman-like + qualities. The spirit of my father grows strong in + me, and I will no longer endure it: therefore allow + me such exercises as may become a gentleman, or + give me the poor allottery my father left me by + testament; with that I will go buy my fortunes. + +OLIVER And what wilt thou do? beg, when that is spent? + Well, sir, get you in: I will not long be troubled + with you; you shall have some part of your will: I + pray you, leave me. + +ORLANDO I will no further offend you than becomes me for my good. + +OLIVER Get you with him, you old dog. + +ADAM Is 'old dog' my reward? Most true, I have lost my + teeth in your service. God be with my old master! + he would not have spoke such a word. + + [Exeunt ORLANDO and ADAM] + +OLIVER Is it even so? begin you to grow upon me? I will + physic your rankness, and yet give no thousand + crowns neither. Holla, Dennis! + + [Enter DENNIS] + +DENNIS Calls your worship? + +OLIVER Was not Charles, the duke's wrestler, here to speak with me? + +DENNIS So please you, he is here at the door and importunes + access to you. + +OLIVER Call him in. + + [Exit DENNIS] + + 'Twill be a good way; and to-morrow the wrestling is. + + [Enter CHARLES] + +CHARLES Good morrow to your worship. + +OLIVER Good Monsieur Charles, what's the new news at the + new court? + +CHARLES There's no news at the court, sir, but the old news: + that is, the old duke is banished by his younger + brother the new duke; and three or four loving lords + have put themselves into voluntary exile with him, + whose lands and revenues enrich the new duke; + therefore he gives them good leave to wander. + +OLIVER Can you tell if Rosalind, the duke's daughter, be + banished with her father? + +CHARLES O, no; for the duke's daughter, her cousin, so loves + her, being ever from their cradles bred together, + that she would have followed her exile, or have died + to stay behind her. She is at the court, and no + less beloved of her uncle than his own daughter; and + never two ladies loved as they do. + +OLIVER Where will the old duke live? + +CHARLES They say he is already in the forest of Arden, and + a many merry men with him; and there they live like + the old Robin Hood of England: they say many young + gentlemen flock to him every day, and fleet the time + carelessly, as they did in the golden world. + +OLIVER What, you wrestle to-morrow before the new duke? + +CHARLES Marry, do I, sir; and I came to acquaint you with a + matter. I am given, sir, secretly to understand + that your younger brother Orlando hath a disposition + to come in disguised against me to try a fall. + To-morrow, sir, I wrestle for my credit; and he that + escapes me without some broken limb shall acquit him + well. Your brother is but young and tender; and, + for your love, I would be loath to foil him, as I + must, for my own honour, if he come in: therefore, + out of my love to you, I came hither to acquaint you + withal, that either you might stay him from his + intendment or brook such disgrace well as he shall + run into, in that it is a thing of his own search + and altogether against my will. + +OLIVER Charles, I thank thee for thy love to me, which + thou shalt find I will most kindly requite. I had + myself notice of my brother's purpose herein and + have by underhand means laboured to dissuade him from + it, but he is resolute. I'll tell thee, Charles: + it is the stubbornest young fellow of France, full + of ambition, an envious emulator of every man's + good parts, a secret and villanous contriver against + me his natural brother: therefore use thy + discretion; I had as lief thou didst break his neck + as his finger. And thou wert best look to't; for if + thou dost him any slight disgrace or if he do not + mightily grace himself on thee, he will practise + against thee by poison, entrap thee by some + treacherous device and never leave thee till he + hath ta'en thy life by some indirect means or other; + for, I assure thee, and almost with tears I speak + it, there is not one so young and so villanous this + day living. I speak but brotherly of him; but + should I anatomize him to thee as he is, I must + blush and weep and thou must look pale and wonder. + +CHARLES I am heartily glad I came hither to you. If he come + to-morrow, I'll give him his payment: if ever he go + alone again, I'll never wrestle for prize more: and + so God keep your worship! + +OLIVER Farewell, good Charles. + + [Exit CHARLES] + + Now will I stir this gamester: I hope I shall see + an end of him; for my soul, yet I know not why, + hates nothing more than he. Yet he's gentle, never + schooled and yet learned, full of noble device, of + all sorts enchantingly beloved, and indeed so much + in the heart of the world, and especially of my own + people, who best know him, that I am altogether + misprised: but it shall not be so long; this + wrestler shall clear all: nothing remains but that + I kindle the boy thither; which now I'll go about. + + [Exit] + + + + + AS YOU LIKE IT + + +ACT I + + + +SCENE II Lawn before the Duke's palace. + + + [Enter CELIA and ROSALIND] + +CELIA I pray thee, Rosalind, sweet my coz, be merry. + +ROSALIND Dear Celia, I show more mirth than I am mistress of; + and would you yet I were merrier? Unless you could + teach me to forget a banished father, you must not + learn me how to remember any extraordinary pleasure. + +CELIA Herein I see thou lovest me not with the full weight + that I love thee. If my uncle, thy banished father, + had banished thy uncle, the duke my father, so thou + hadst been still with me, I could have taught my + love to take thy father for mine: so wouldst thou, + if the truth of thy love to me were so righteously + tempered as mine is to thee. + +ROSALIND Well, I will forget the condition of my estate, to + rejoice in yours. + +CELIA You know my father hath no child but I, nor none is + like to have: and, truly, when he dies, thou shalt + be his heir, for what he hath taken away from thy + father perforce, I will render thee again in + affection; by mine honour, I will; and when I break + that oath, let me turn monster: therefore, my + sweet Rose, my dear Rose, be merry. + +ROSALIND From henceforth I will, coz, and devise sports. Let + me see; what think you of falling in love? + +CELIA Marry, I prithee, do, to make sport withal: but + love no man in good earnest; nor no further in sport + neither than with safety of a pure blush thou mayst + in honour come off again. + +ROSALIND What shall be our sport, then? + +CELIA Let us sit and mock the good housewife Fortune from + her wheel, that her gifts may henceforth be bestowed equally. + +ROSALIND I would we could do so, for her benefits are + mightily misplaced, and the bountiful blind woman + doth most mistake in her gifts to women. + +CELIA 'Tis true; for those that she makes fair she scarce + makes honest, and those that she makes honest she + makes very ill-favouredly. + +ROSALIND Nay, now thou goest from Fortune's office to + Nature's: Fortune reigns in gifts of the world, + not in the lineaments of Nature. + + [Enter TOUCHSTONE] + +CELIA No? when Nature hath made a fair creature, may she + not by Fortune fall into the fire? Though Nature + hath given us wit to flout at Fortune, hath not + Fortune sent in this fool to cut off the argument? + +ROSALIND Indeed, there is Fortune too hard for Nature, when + Fortune makes Nature's natural the cutter-off of + Nature's wit. + +CELIA Peradventure this is not Fortune's work neither, but + Nature's; who perceiveth our natural wits too dull + to reason of such goddesses and hath sent this + natural for our whetstone; for always the dulness of + the fool is the whetstone of the wits. How now, + wit! whither wander you? + +TOUCHSTONE Mistress, you must come away to your father. + +CELIA Were you made the messenger? + +TOUCHSTONE No, by mine honour, but I was bid to come for you. + +ROSALIND Where learned you that oath, fool? + +TOUCHSTONE Of a certain knight that swore by his honour they + were good pancakes and swore by his honour the + mustard was naught: now I'll stand to it, the + pancakes were naught and the mustard was good, and + yet was not the knight forsworn. + +CELIA How prove you that, in the great heap of your + knowledge? + +ROSALIND Ay, marry, now unmuzzle your wisdom. + +TOUCHSTONE Stand you both forth now: stroke your chins, and + swear by your beards that I am a knave. + +CELIA By our beards, if we had them, thou art. + +TOUCHSTONE By my knavery, if I had it, then I were; but if you + swear by that that is not, you are not forsworn: no + more was this knight swearing by his honour, for he + never had any; or if he had, he had sworn it away + before ever he saw those pancakes or that mustard. + +CELIA Prithee, who is't that thou meanest? + +TOUCHSTONE One that old Frederick, your father, loves. + +CELIA My father's love is enough to honour him: enough! + speak no more of him; you'll be whipped for taxation + one of these days. + +TOUCHSTONE The more pity, that fools may not speak wisely what + wise men do foolishly. + +CELIA By my troth, thou sayest true; for since the little + wit that fools have was silenced, the little foolery + that wise men have makes a great show. Here comes + Monsieur Le Beau. + +ROSALIND With his mouth full of news. + +CELIA Which he will put on us, as pigeons feed their young. + +ROSALIND Then shall we be news-crammed. + +CELIA All the better; we shall be the more marketable. + + [Enter LE BEAU] + + Bon jour, Monsieur Le Beau: what's the news? + +LE BEAU Fair princess, you have lost much good sport. + +CELIA Sport! of what colour? + +LE BEAU What colour, madam! how shall I answer you? + +ROSALIND As wit and fortune will. + +TOUCHSTONE Or as the Destinies decree. + +CELIA Well said: that was laid on with a trowel. + +TOUCHSTONE Nay, if I keep not my rank,-- + +ROSALIND Thou losest thy old smell. + +LE BEAU You amaze me, ladies: I would have told you of good + wrestling, which you have lost the sight of. + +ROSALIND You tell us the manner of the wrestling. + +LE BEAU I will tell you the beginning; and, if it please + your ladyships, you may see the end; for the best is + yet to do; and here, where you are, they are coming + to perform it. + +CELIA Well, the beginning, that is dead and buried. + +LE BEAU There comes an old man and his three sons,-- + +CELIA I could match this beginning with an old tale. + +LE BEAU Three proper young men, of excellent growth and presence. + +ROSALIND With bills on their necks, 'Be it known unto all men + by these presents.' + +LE BEAU The eldest of the three wrestled with Charles, the + duke's wrestler; which Charles in a moment threw him + and broke three of his ribs, that there is little + hope of life in him: so he served the second, and + so the third. Yonder they lie; the poor old man, + their father, making such pitiful dole over them + that all the beholders take his part with weeping. + +ROSALIND Alas! + +TOUCHSTONE But what is the sport, monsieur, that the ladies + have lost? + +LE BEAU Why, this that I speak of. + +TOUCHSTONE Thus men may grow wiser every day: it is the first + time that ever I heard breaking of ribs was sport + for ladies. + +CELIA Or I, I promise thee. + +ROSALIND But is there any else longs to see this broken music + in his sides? is there yet another dotes upon + rib-breaking? Shall we see this wrestling, cousin? + +LE BEAU You must, if you stay here; for here is the place + appointed for the wrestling, and they are ready to + perform it. + +CELIA Yonder, sure, they are coming: let us now stay and see it. + + [Flourish. Enter DUKE FREDERICK, Lords, ORLANDO, + CHARLES, and Attendants] + +DUKE FREDERICK Come on: since the youth will not be entreated, his + own peril on his forwardness. + +ROSALIND Is yonder the man? + +LE BEAU Even he, madam. + +CELIA Alas, he is too young! yet he looks successfully. + +DUKE FREDERICK How now, daughter and cousin! are you crept hither + to see the wrestling? + +ROSALIND Ay, my liege, so please you give us leave. + +DUKE FREDERICK You will take little delight in it, I can tell you; + there is such odds in the man. In pity of the + challenger's youth I would fain dissuade him, but he + will not be entreated. Speak to him, ladies; see if + you can move him. + +CELIA Call him hither, good Monsieur Le Beau. + +DUKE FREDERICK Do so: I'll not be by. + +LE BEAU Monsieur the challenger, the princesses call for you. + +ORLANDO I attend them with all respect and duty. + +ROSALIND Young man, have you challenged Charles the wrestler? + +ORLANDO No, fair princess; he is the general challenger: I + come but in, as others do, to try with him the + strength of my youth. + +CELIA Young gentleman, your spirits are too bold for your + years. You have seen cruel proof of this man's + strength: if you saw yourself with your eyes or + knew yourself with your judgment, the fear of your + adventure would counsel you to a more equal + enterprise. We pray you, for your own sake, to + embrace your own safety and give over this attempt. + +ROSALIND Do, young sir; your reputation shall not therefore + be misprised: we will make it our suit to the duke + that the wrestling might not go forward. + +ORLANDO I beseech you, punish me not with your hard + thoughts; wherein I confess me much guilty, to deny + so fair and excellent ladies any thing. But let + your fair eyes and gentle wishes go with me to my + trial: wherein if I be foiled, there is but one + shamed that was never gracious; if killed, but one + dead that was willing to be so: I shall do my + friends no wrong, for I have none to lament me, the + world no injury, for in it I have nothing; only in + the world I fill up a place, which may be better + supplied when I have made it empty. + +ROSALIND The little strength that I have, I would it were with you. + +CELIA And mine, to eke out hers. + +ROSALIND Fare you well: pray heaven I be deceived in you! + +CELIA Your heart's desires be with you! + +CHARLES Come, where is this young gallant that is so + desirous to lie with his mother earth? + +ORLANDO Ready, sir; but his will hath in it a more modest working. + +DUKE FREDERICK You shall try but one fall. + +CHARLES No, I warrant your grace, you shall not entreat him + to a second, that have so mightily persuaded him + from a first. + +ORLANDO An you mean to mock me after, you should not have + mocked me before: but come your ways. + +ROSALIND Now Hercules be thy speed, young man! + +CELIA I would I were invisible, to catch the strong + fellow by the leg. + + [They wrestle] + +ROSALIND O excellent young man! + +CELIA If I had a thunderbolt in mine eye, I can tell who + should down. + + [Shout. CHARLES is thrown] + +DUKE FREDERICK No more, no more. + +ORLANDO Yes, I beseech your grace: I am not yet well breathed. + +DUKE FREDERICK How dost thou, Charles? + +LE BEAU He cannot speak, my lord. + +DUKE FREDERICK Bear him away. What is thy name, young man? + +ORLANDO Orlando, my liege; the youngest son of Sir Rowland de Boys. + +DUKE FREDERICK I would thou hadst been son to some man else: + The world esteem'd thy father honourable, + But I did find him still mine enemy: + Thou shouldst have better pleased me with this deed, + Hadst thou descended from another house. + But fare thee well; thou art a gallant youth: + I would thou hadst told me of another father. + + [Exeunt DUKE FREDERICK, train, and LE BEAU] + +CELIA Were I my father, coz, would I do this? + +ORLANDO I am more proud to be Sir Rowland's son, + His youngest son; and would not change that calling, + To be adopted heir to Frederick. + +ROSALIND My father loved Sir Rowland as his soul, + And all the world was of my father's mind: + Had I before known this young man his son, + I should have given him tears unto entreaties, + Ere he should thus have ventured. + +CELIA Gentle cousin, + Let us go thank him and encourage him: + My father's rough and envious disposition + Sticks me at heart. Sir, you have well deserved: + If you do keep your promises in love + But justly, as you have exceeded all promise, + Your mistress shall be happy. + +ROSALIND Gentleman, + + [Giving him a chain from her neck] + + Wear this for me, one out of suits with fortune, + That could give more, but that her hand lacks means. + Shall we go, coz? + +CELIA Ay. Fare you well, fair gentleman. + +ORLANDO Can I not say, I thank you? My better parts + Are all thrown down, and that which here stands up + Is but a quintain, a mere lifeless block. + +ROSALIND He calls us back: my pride fell with my fortunes; + I'll ask him what he would. Did you call, sir? + Sir, you have wrestled well and overthrown + More than your enemies. + +CELIA Will you go, coz? + +ROSALIND Have with you. Fare you well. + + [Exeunt ROSALIND and CELIA] + +ORLANDO What passion hangs these weights upon my tongue? + I cannot speak to her, yet she urged conference. + O poor Orlando, thou art overthrown! + Or Charles or something weaker masters thee. + + [Re-enter LE BEAU] + +LE BEAU Good sir, I do in friendship counsel you + To leave this place. Albeit you have deserved + High commendation, true applause and love, + Yet such is now the duke's condition + That he misconstrues all that you have done. + The duke is humorous; what he is indeed, + More suits you to conceive than I to speak of. + +ORLANDO I thank you, sir: and, pray you, tell me this: + Which of the two was daughter of the duke + That here was at the wrestling? + +LE BEAU Neither his daughter, if we judge by manners; + But yet indeed the lesser is his daughter + The other is daughter to the banish'd duke, + And here detain'd by her usurping uncle, + To keep his daughter company; whose loves + Are dearer than the natural bond of sisters. + But I can tell you that of late this duke + Hath ta'en displeasure 'gainst his gentle niece, + Grounded upon no other argument + But that the people praise her for her virtues + And pity her for her good father's sake; + And, on my life, his malice 'gainst the lady + Will suddenly break forth. Sir, fare you well: + Hereafter, in a better world than this, + I shall desire more love and knowledge of you. + +ORLANDO I rest much bounden to you: fare you well. + + [Exit LE BEAU] + + Thus must I from the smoke into the smother; + From tyrant duke unto a tyrant brother: + But heavenly Rosalind! + + [Exit] + + + + + AS YOU LIKE IT + + +ACT I + + + +SCENE III A room in the palace. + + + [Enter CELIA and ROSALIND] + +CELIA Why, cousin! why, Rosalind! Cupid have mercy! not a word? + +ROSALIND Not one to throw at a dog. + +CELIA No, thy words are too precious to be cast away upon + curs; throw some of them at me; come, lame me with reasons. + +ROSALIND Then there were two cousins laid up; when the one + should be lamed with reasons and the other mad + without any. + +CELIA But is all this for your father? + +ROSALIND No, some of it is for my child's father. O, how + full of briers is this working-day world! + +CELIA They are but burs, cousin, thrown upon thee in + holiday foolery: if we walk not in the trodden + paths our very petticoats will catch them. + +ROSALIND I could shake them off my coat: these burs are in my heart. + +CELIA Hem them away. + +ROSALIND I would try, if I could cry 'hem' and have him. + +CELIA Come, come, wrestle with thy affections. + +ROSALIND O, they take the part of a better wrestler than myself! + +CELIA O, a good wish upon you! you will try in time, in + despite of a fall. But, turning these jests out of + service, let us talk in good earnest: is it + possible, on such a sudden, you should fall into so + strong a liking with old Sir Rowland's youngest son? + +ROSALIND The duke my father loved his father dearly. + +CELIA Doth it therefore ensue that you should love his son + dearly? By this kind of chase, I should hate him, + for my father hated his father dearly; yet I hate + not Orlando. + +ROSALIND No, faith, hate him not, for my sake. + +CELIA Why should I not? doth he not deserve well? + +ROSALIND Let me love him for that, and do you love him + because I do. Look, here comes the duke. + +CELIA With his eyes full of anger. + + [Enter DUKE FREDERICK, with Lords] + +DUKE FREDERICK Mistress, dispatch you with your safest haste + And get you from our court. + +ROSALIND Me, uncle? + +DUKE FREDERICK You, cousin + Within these ten days if that thou be'st found + So near our public court as twenty miles, + Thou diest for it. + +ROSALIND I do beseech your grace, + Let me the knowledge of my fault bear with me: + If with myself I hold intelligence + Or have acquaintance with mine own desires, + If that I do not dream or be not frantic,-- + As I do trust I am not--then, dear uncle, + Never so much as in a thought unborn + Did I offend your highness. + +DUKE FREDERICK Thus do all traitors: + If their purgation did consist in words, + They are as innocent as grace itself: + Let it suffice thee that I trust thee not. + +ROSALIND Yet your mistrust cannot make me a traitor: + Tell me whereon the likelihood depends. + +DUKE FREDERICK Thou art thy father's daughter; there's enough. + +ROSALIND So was I when your highness took his dukedom; + So was I when your highness banish'd him: + Treason is not inherited, my lord; + Or, if we did derive it from our friends, + What's that to me? my father was no traitor: + Then, good my liege, mistake me not so much + To think my poverty is treacherous. + +CELIA Dear sovereign, hear me speak. + +DUKE FREDERICK Ay, Celia; we stay'd her for your sake, + Else had she with her father ranged along. + +CELIA I did not then entreat to have her stay; + It was your pleasure and your own remorse: + I was too young that time to value her; + But now I know her: if she be a traitor, + Why so am I; we still have slept together, + Rose at an instant, learn'd, play'd, eat together, + And wheresoever we went, like Juno's swans, + Still we went coupled and inseparable. + +DUKE FREDERICK She is too subtle for thee; and her smoothness, + Her very silence and her patience + Speak to the people, and they pity her. + Thou art a fool: she robs thee of thy name; + And thou wilt show more bright and seem more virtuous + When she is gone. Then open not thy lips: + Firm and irrevocable is my doom + Which I have pass'd upon her; she is banish'd. + +CELIA Pronounce that sentence then on me, my liege: + I cannot live out of her company. + +DUKE FREDERICK You are a fool. You, niece, provide yourself: + If you outstay the time, upon mine honour, + And in the greatness of my word, you die. + + [Exeunt DUKE FREDERICK and Lords] + +CELIA O my poor Rosalind, whither wilt thou go? + Wilt thou change fathers? I will give thee mine. + I charge thee, be not thou more grieved than I am. + +ROSALIND I have more cause. + +CELIA Thou hast not, cousin; + Prithee be cheerful: know'st thou not, the duke + Hath banish'd me, his daughter? + +ROSALIND That he hath not. + +CELIA No, hath not? Rosalind lacks then the love + Which teacheth thee that thou and I am one: + Shall we be sunder'd? shall we part, sweet girl? + No: let my father seek another heir. + Therefore devise with me how we may fly, + Whither to go and what to bear with us; + And do not seek to take your change upon you, + To bear your griefs yourself and leave me out; + For, by this heaven, now at our sorrows pale, + Say what thou canst, I'll go along with thee. + +ROSALIND Why, whither shall we go? + +CELIA To seek my uncle in the forest of Arden. + +ROSALIND Alas, what danger will it be to us, + Maids as we are, to travel forth so far! + Beauty provoketh thieves sooner than gold. + +CELIA I'll put myself in poor and mean attire + And with a kind of umber smirch my face; + The like do you: so shall we pass along + And never stir assailants. + +ROSALIND Were it not better, + Because that I am more than common tall, + That I did suit me all points like a man? + A gallant curtle-axe upon my thigh, + A boar-spear in my hand; and--in my heart + Lie there what hidden woman's fear there will-- + We'll have a swashing and a martial outside, + As many other mannish cowards have + That do outface it with their semblances. + +CELIA What shall I call thee when thou art a man? + +ROSALIND I'll have no worse a name than Jove's own page; + And therefore look you call me Ganymede. + But what will you be call'd? + +CELIA Something that hath a reference to my state + No longer Celia, but Aliena. + +ROSALIND But, cousin, what if we assay'd to steal + The clownish fool out of your father's court? + Would he not be a comfort to our travel? + +CELIA He'll go along o'er the wide world with me; + Leave me alone to woo him. Let's away, + And get our jewels and our wealth together, + Devise the fittest time and safest way + To hide us from pursuit that will be made + After my flight. Now go we in content + To liberty and not to banishment. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT II + + + +SCENE I The Forest of Arden. + + + [Enter DUKE SENIOR, AMIENS, and two or three Lords, + like foresters] + +DUKE SENIOR Now, my co-mates and brothers in exile, + Hath not old custom made this life more sweet + Than that of painted pomp? Are not these woods + More free from peril than the envious court? + Here feel we but the penalty of Adam, + The seasons' difference, as the icy fang + And churlish chiding of the winter's wind, + Which, when it bites and blows upon my body, + Even till I shrink with cold, I smile and say + 'This is no flattery: these are counsellors + That feelingly persuade me what I am.' + Sweet are the uses of adversity, + Which, like the toad, ugly and venomous, + Wears yet a precious jewel in his head; + And this our life exempt from public haunt + Finds tongues in trees, books in the running brooks, + Sermons in stones and good in every thing. + I would not change it. + +AMIENS Happy is your grace, + That can translate the stubbornness of fortune + Into so quiet and so sweet a style. + +DUKE SENIOR Come, shall we go and kill us venison? + And yet it irks me the poor dappled fools, + Being native burghers of this desert city, + Should in their own confines with forked heads + Have their round haunches gored. + +First Lord Indeed, my lord, + The melancholy Jaques grieves at that, + And, in that kind, swears you do more usurp + Than doth your brother that hath banish'd you. + To-day my Lord of Amiens and myself + Did steal behind him as he lay along + Under an oak whose antique root peeps out + Upon the brook that brawls along this wood: + To the which place a poor sequester'd stag, + That from the hunter's aim had ta'en a hurt, + Did come to languish, and indeed, my lord, + The wretched animal heaved forth such groans + That their discharge did stretch his leathern coat + Almost to bursting, and the big round tears + Coursed one another down his innocent nose + In piteous chase; and thus the hairy fool + Much marked of the melancholy Jaques, + Stood on the extremest verge of the swift brook, + Augmenting it with tears. + +DUKE SENIOR But what said Jaques? + Did he not moralize this spectacle? + +First Lord O, yes, into a thousand similes. + First, for his weeping into the needless stream; + 'Poor deer,' quoth he, 'thou makest a testament + As worldlings do, giving thy sum of more + To that which had too much:' then, being there alone, + Left and abandon'd of his velvet friends, + ''Tis right:' quoth he; 'thus misery doth part + The flux of company:' anon a careless herd, + Full of the pasture, jumps along by him + And never stays to greet him; 'Ay' quoth Jaques, + 'Sweep on, you fat and greasy citizens; + 'Tis just the fashion: wherefore do you look + Upon that poor and broken bankrupt there?' + Thus most invectively he pierceth through + The body of the country, city, court, + Yea, and of this our life, swearing that we + Are mere usurpers, tyrants and what's worse, + To fright the animals and to kill them up + In their assign'd and native dwelling-place. + +DUKE SENIOR And did you leave him in this contemplation? + +Second Lord We did, my lord, weeping and commenting + Upon the sobbing deer. + +DUKE SENIOR Show me the place: + I love to cope him in these sullen fits, + For then he's full of matter. + +First Lord I'll bring you to him straight. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT II + + + +SCENE II A room in the palace. + + + [Enter DUKE FREDERICK, with Lords] + +DUKE FREDERICK Can it be possible that no man saw them? + It cannot be: some villains of my court + Are of consent and sufferance in this. + +First Lord I cannot hear of any that did see her. + The ladies, her attendants of her chamber, + Saw her abed, and in the morning early + They found the bed untreasured of their mistress. + +Second Lord My lord, the roynish clown, at whom so oft + Your grace was wont to laugh, is also missing. + Hisperia, the princess' gentlewoman, + Confesses that she secretly o'erheard + Your daughter and her cousin much commend + The parts and graces of the wrestler + That did but lately foil the sinewy Charles; + And she believes, wherever they are gone, + That youth is surely in their company. + +DUKE FREDERICK Send to his brother; fetch that gallant hither; + If he be absent, bring his brother to me; + I'll make him find him: do this suddenly, + And let not search and inquisition quail + To bring again these foolish runaways. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT II + + + +SCENE III Before OLIVER'S house. + + + [Enter ORLANDO and ADAM, meeting] + +ORLANDO Who's there? + +ADAM What, my young master? O, my gentle master! + O my sweet master! O you memory + Of old Sir Rowland! why, what make you here? + Why are you virtuous? why do people love you? + And wherefore are you gentle, strong and valiant? + Why would you be so fond to overcome + The bonny priser of the humorous duke? + Your praise is come too swiftly home before you. + Know you not, master, to some kind of men + Their graces serve them but as enemies? + No more do yours: your virtues, gentle master, + Are sanctified and holy traitors to you. + O, what a world is this, when what is comely + Envenoms him that bears it! + +ORLANDO Why, what's the matter? + +ADAM O unhappy youth! + Come not within these doors; within this roof + The enemy of all your graces lives: + Your brother--no, no brother; yet the son-- + Yet not the son, I will not call him son + Of him I was about to call his father-- + Hath heard your praises, and this night he means + To burn the lodging where you use to lie + And you within it: if he fail of that, + He will have other means to cut you off. + I overheard him and his practises. + This is no place; this house is but a butchery: + Abhor it, fear it, do not enter it. + +ORLANDO Why, whither, Adam, wouldst thou have me go? + +ADAM No matter whither, so you come not here. + +ORLANDO What, wouldst thou have me go and beg my food? + Or with a base and boisterous sword enforce + A thievish living on the common road? + This I must do, or know not what to do: + Yet this I will not do, do how I can; + I rather will subject me to the malice + Of a diverted blood and bloody brother. + +ADAM But do not so. I have five hundred crowns, + The thrifty hire I saved under your father, + Which I did store to be my foster-nurse + When service should in my old limbs lie lame + And unregarded age in corners thrown: + Take that, and He that doth the ravens feed, + Yea, providently caters for the sparrow, + Be comfort to my age! Here is the gold; + And all this I give you. Let me be your servant: + Though I look old, yet I am strong and lusty; + For in my youth I never did apply + Hot and rebellious liquors in my blood, + Nor did not with unbashful forehead woo + The means of weakness and debility; + Therefore my age is as a lusty winter, + Frosty, but kindly: let me go with you; + I'll do the service of a younger man + In all your business and necessities. + +ORLANDO O good old man, how well in thee appears + The constant service of the antique world, + When service sweat for duty, not for meed! + Thou art not for the fashion of these times, + Where none will sweat but for promotion, + And having that, do choke their service up + Even with the having: it is not so with thee. + But, poor old man, thou prunest a rotten tree, + That cannot so much as a blossom yield + In lieu of all thy pains and husbandry + But come thy ways; well go along together, + And ere we have thy youthful wages spent, + We'll light upon some settled low content. + +ADAM Master, go on, and I will follow thee, + To the last gasp, with truth and loyalty. + From seventeen years till now almost fourscore + Here lived I, but now live here no more. + At seventeen years many their fortunes seek; + But at fourscore it is too late a week: + Yet fortune cannot recompense me better + Than to die well and not my master's debtor. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT II + + + +SCENE IV The Forest of Arden. + + + [Enter ROSALIND for Ganymede, CELIA for Aliena, + and TOUCHSTONE] + +ROSALIND O Jupiter, how weary are my spirits! + +TOUCHSTONE I care not for my spirits, if my legs were not weary. + +ROSALIND I could find in my heart to disgrace my man's + apparel and to cry like a woman; but I must comfort + the weaker vessel, as doublet and hose ought to show + itself courageous to petticoat: therefore courage, + good Aliena! + +CELIA I pray you, bear with me; I cannot go no further. + +TOUCHSTONE For my part, I had rather bear with you than bear + you; yet I should bear no cross if I did bear you, + for I think you have no money in your purse. + +ROSALIND Well, this is the forest of Arden. + +TOUCHSTONE Ay, now am I in Arden; the more fool I; when I was + at home, I was in a better place: but travellers + must be content. + +ROSALIND Ay, be so, good Touchstone. + + [Enter CORIN and SILVIUS] + + Look you, who comes here; a young man and an old in + solemn talk. + +CORIN That is the way to make her scorn you still. + +SILVIUS O Corin, that thou knew'st how I do love her! + +CORIN I partly guess; for I have loved ere now. + +SILVIUS No, Corin, being old, thou canst not guess, + Though in thy youth thou wast as true a lover + As ever sigh'd upon a midnight pillow: + But if thy love were ever like to mine-- + As sure I think did never man love so-- + How many actions most ridiculous + Hast thou been drawn to by thy fantasy? + +CORIN Into a thousand that I have forgotten. + +SILVIUS O, thou didst then ne'er love so heartily! + If thou remember'st not the slightest folly + That ever love did make thee run into, + Thou hast not loved: + Or if thou hast not sat as I do now, + Wearying thy hearer in thy mistress' praise, + Thou hast not loved: + Or if thou hast not broke from company + Abruptly, as my passion now makes me, + Thou hast not loved. + O Phebe, Phebe, Phebe! + + [Exit] + +ROSALIND Alas, poor shepherd! searching of thy wound, + I have by hard adventure found mine own. + +TOUCHSTONE And I mine. I remember, when I was in love I broke + my sword upon a stone and bid him take that for + coming a-night to Jane Smile; and I remember the + kissing of her batlet and the cow's dugs that her + pretty chopt hands had milked; and I remember the + wooing of a peascod instead of her, from whom I took + two cods and, giving her them again, said with + weeping tears 'Wear these for my sake.' We that are + true lovers run into strange capers; but as all is + mortal in nature, so is all nature in love mortal in folly. + +ROSALIND Thou speakest wiser than thou art ware of. + +TOUCHSTONE Nay, I shall ne'er be ware of mine own wit till I + break my shins against it. + +ROSALIND Jove, Jove! this shepherd's passion + Is much upon my fashion. + +TOUCHSTONE And mine; but it grows something stale with me. + +CELIA I pray you, one of you question yond man + If he for gold will give us any food: + I faint almost to death. + +TOUCHSTONE Holla, you clown! + +ROSALIND Peace, fool: he's not thy kinsman. + +CORIN Who calls? + +TOUCHSTONE Your betters, sir. + +CORIN Else are they very wretched. + +ROSALIND Peace, I say. Good even to you, friend. + +CORIN And to you, gentle sir, and to you all. + +ROSALIND I prithee, shepherd, if that love or gold + Can in this desert place buy entertainment, + Bring us where we may rest ourselves and feed: + Here's a young maid with travel much oppress'd + And faints for succor. + +CORIN Fair sir, I pity her + And wish, for her sake more than for mine own, + My fortunes were more able to relieve her; + But I am shepherd to another man + And do not shear the fleeces that I graze: + My master is of churlish disposition + And little recks to find the way to heaven + By doing deeds of hospitality: + Besides, his cote, his flocks and bounds of feed + Are now on sale, and at our sheepcote now, + By reason of his absence, there is nothing + That you will feed on; but what is, come see. + And in my voice most welcome shall you be. + +ROSALIND What is he that shall buy his flock and pasture? + +CORIN That young swain that you saw here but erewhile, + That little cares for buying any thing. + +ROSALIND I pray thee, if it stand with honesty, + Buy thou the cottage, pasture and the flock, + And thou shalt have to pay for it of us. + +CELIA And we will mend thy wages. I like this place. + And willingly could waste my time in it. + +CORIN Assuredly the thing is to be sold: + Go with me: if you like upon report + The soil, the profit and this kind of life, + I will your very faithful feeder be + And buy it with your gold right suddenly. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT II + + + +SCENE V The Forest. + + + [Enter AMIENS, JAQUES, and others] + + SONG. +AMIENS Under the greenwood tree + Who loves to lie with me, + And turn his merry note + Unto the sweet bird's throat, + Come hither, come hither, come hither: + Here shall he see No enemy + But winter and rough weather. + +JAQUES More, more, I prithee, more. + +AMIENS It will make you melancholy, Monsieur Jaques. + +JAQUES I thank it. More, I prithee, more. I can suck + melancholy out of a song, as a weasel sucks eggs. + More, I prithee, more. + +AMIENS My voice is ragged: I know I cannot please you. + +JAQUES I do not desire you to please me; I do desire you to + sing. Come, more; another stanzo: call you 'em stanzos? + +AMIENS What you will, Monsieur Jaques. + +JAQUES Nay, I care not for their names; they owe me + nothing. Will you sing? + +AMIENS More at your request than to please myself. + +JAQUES Well then, if ever I thank any man, I'll thank you; + but that they call compliment is like the encounter + of two dog-apes, and when a man thanks me heartily, + methinks I have given him a penny and he renders me + the beggarly thanks. Come, sing; and you that will + not, hold your tongues. + +AMIENS Well, I'll end the song. Sirs, cover the while; the + duke will drink under this tree. He hath been all + this day to look you. + +JAQUES And I have been all this day to avoid him. He is + too disputable for my company: I think of as many + matters as he, but I give heaven thanks and make no + boast of them. Come, warble, come. + + SONG. + Who doth ambition shun + + [All together here] + + And loves to live i' the sun, + Seeking the food he eats + And pleased with what he gets, + Come hither, come hither, come hither: + Here shall he see No enemy + But winter and rough weather. + +JAQUES I'll give you a verse to this note that I made + yesterday in despite of my invention. + +AMIENS And I'll sing it. + +JAQUES Thus it goes:-- + + If it do come to pass + That any man turn ass, + Leaving his wealth and ease, + A stubborn will to please, + Ducdame, ducdame, ducdame: + Here shall he see + Gross fools as he, + An if he will come to me. + +AMIENS What's that 'ducdame'? + +JAQUES 'Tis a Greek invocation, to call fools into a + circle. I'll go sleep, if I can; if I cannot, I'll + rail against all the first-born of Egypt. + +AMIENS And I'll go seek the duke: his banquet is prepared. + + [Exeunt severally] + + + + + AS YOU LIKE IT + + +ACT II + + + +SCENE VI The forest. + + + [Enter ORLANDO and ADAM] + +ADAM Dear master, I can go no further. O, I die for food! + Here lie I down, and measure out my grave. Farewell, + kind master. + +ORLANDO Why, how now, Adam! no greater heart in thee? Live + a little; comfort a little; cheer thyself a little. + If this uncouth forest yield any thing savage, I + will either be food for it or bring it for food to + thee. Thy conceit is nearer death than thy powers. + For my sake be comfortable; hold death awhile at + the arm's end: I will here be with thee presently; + and if I bring thee not something to eat, I will + give thee leave to die: but if thou diest before I + come, thou art a mocker of my labour. Well said! + thou lookest cheerly, and I'll be with thee quickly. + Yet thou liest in the bleak air: come, I will bear + thee to some shelter; and thou shalt not die for + lack of a dinner, if there live any thing in this + desert. Cheerly, good Adam! + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT II + + + +SCENE VII The forest. + + + [A table set out. Enter DUKE SENIOR, AMIENS, and + Lords like outlaws] + +DUKE SENIOR I think he be transform'd into a beast; + For I can no where find him like a man. + +First Lord My lord, he is but even now gone hence: + Here was he merry, hearing of a song. + +DUKE SENIOR If he, compact of jars, grow musical, + We shall have shortly discord in the spheres. + Go, seek him: tell him I would speak with him. + + [Enter JAQUES] + +First Lord He saves my labour by his own approach. + +DUKE SENIOR Why, how now, monsieur! what a life is this, + That your poor friends must woo your company? + What, you look merrily! + +JAQUES A fool, a fool! I met a fool i' the forest, + A motley fool; a miserable world! + As I do live by food, I met a fool + Who laid him down and bask'd him in the sun, + And rail'd on Lady Fortune in good terms, + In good set terms and yet a motley fool. + 'Good morrow, fool,' quoth I. 'No, sir,' quoth he, + 'Call me not fool till heaven hath sent me fortune:' + And then he drew a dial from his poke, + And, looking on it with lack-lustre eye, + Says very wisely, 'It is ten o'clock: + Thus we may see,' quoth he, 'how the world wags: + 'Tis but an hour ago since it was nine, + And after one hour more 'twill be eleven; + And so, from hour to hour, we ripe and ripe, + And then, from hour to hour, we rot and rot; + And thereby hangs a tale.' When I did hear + The motley fool thus moral on the time, + My lungs began to crow like chanticleer, + That fools should be so deep-contemplative, + And I did laugh sans intermission + An hour by his dial. O noble fool! + A worthy fool! Motley's the only wear. + +DUKE SENIOR What fool is this? + +JAQUES O worthy fool! One that hath been a courtier, + And says, if ladies be but young and fair, + They have the gift to know it: and in his brain, + Which is as dry as the remainder biscuit + After a voyage, he hath strange places cramm'd + With observation, the which he vents + In mangled forms. O that I were a fool! + I am ambitious for a motley coat. + +DUKE SENIOR Thou shalt have one. + +JAQUES It is my only suit; + Provided that you weed your better judgments + Of all opinion that grows rank in them + That I am wise. I must have liberty + Withal, as large a charter as the wind, + To blow on whom I please; for so fools have; + And they that are most galled with my folly, + They most must laugh. And why, sir, must they so? + The 'why' is plain as way to parish church: + He that a fool doth very wisely hit + Doth very foolishly, although he smart, + Not to seem senseless of the bob: if not, + The wise man's folly is anatomized + Even by the squandering glances of the fool. + Invest me in my motley; give me leave + To speak my mind, and I will through and through + Cleanse the foul body of the infected world, + If they will patiently receive my medicine. + +DUKE SENIOR Fie on thee! I can tell what thou wouldst do. + +JAQUES What, for a counter, would I do but good? + +DUKE SENIOR Most mischievous foul sin, in chiding sin: + For thou thyself hast been a libertine, + As sensual as the brutish sting itself; + And all the embossed sores and headed evils, + That thou with licence of free foot hast caught, + Wouldst thou disgorge into the general world. + +JAQUES Why, who cries out on pride, + That can therein tax any private party? + Doth it not flow as hugely as the sea, + Till that the weary very means do ebb? + What woman in the city do I name, + When that I say the city-woman bears + The cost of princes on unworthy shoulders? + Who can come in and say that I mean her, + When such a one as she such is her neighbour? + Or what is he of basest function + That says his bravery is not of my cost, + Thinking that I mean him, but therein suits + His folly to the mettle of my speech? + There then; how then? what then? Let me see wherein + My tongue hath wrong'd him: if it do him right, + Then he hath wrong'd himself; if he be free, + Why then my taxing like a wild-goose flies, + Unclaim'd of any man. But who comes here? + + [Enter ORLANDO, with his sword drawn] + +ORLANDO Forbear, and eat no more. + +JAQUES Why, I have eat none yet. + +ORLANDO Nor shalt not, till necessity be served. + +JAQUES Of what kind should this cock come of? + +DUKE SENIOR Art thou thus bolden'd, man, by thy distress, + Or else a rude despiser of good manners, + That in civility thou seem'st so empty? + +ORLANDO You touch'd my vein at first: the thorny point + Of bare distress hath ta'en from me the show + Of smooth civility: yet am I inland bred + And know some nurture. But forbear, I say: + He dies that touches any of this fruit + Till I and my affairs are answered. + +JAQUES An you will not be answered with reason, I must die. + +DUKE SENIOR What would you have? Your gentleness shall force + More than your force move us to gentleness. + +ORLANDO I almost die for food; and let me have it. + +DUKE SENIOR Sit down and feed, and welcome to our table. + +ORLANDO Speak you so gently? Pardon me, I pray you: + I thought that all things had been savage here; + And therefore put I on the countenance + Of stern commandment. But whate'er you are + That in this desert inaccessible, + Under the shade of melancholy boughs, + Lose and neglect the creeping hours of time + If ever you have look'd on better days, + If ever been where bells have knoll'd to church, + If ever sat at any good man's feast, + If ever from your eyelids wiped a tear + And know what 'tis to pity and be pitied, + Let gentleness my strong enforcement be: + In the which hope I blush, and hide my sword. + +DUKE SENIOR True is it that we have seen better days, + And have with holy bell been knoll'd to church + And sat at good men's feasts and wiped our eyes + Of drops that sacred pity hath engender'd: + And therefore sit you down in gentleness + And take upon command what help we have + That to your wanting may be minister'd. + +ORLANDO Then but forbear your food a little while, + Whiles, like a doe, I go to find my fawn + And give it food. There is an old poor man, + Who after me hath many a weary step + Limp'd in pure love: till he be first sufficed, + Oppress'd with two weak evils, age and hunger, + I will not touch a bit. + +DUKE SENIOR Go find him out, + And we will nothing waste till you return. + +ORLANDO I thank ye; and be blest for your good comfort! + + [Exit] + +DUKE SENIOR Thou seest we are not all alone unhappy: + This wide and universal theatre + Presents more woeful pageants than the scene + Wherein we play in. + +JAQUES All the world's a stage, + And all the men and women merely players: + They have their exits and their entrances; + And one man in his time plays many parts, + His acts being seven ages. At first the infant, + Mewling and puking in the nurse's arms. + And then the whining school-boy, with his satchel + And shining morning face, creeping like snail + Unwillingly to school. And then the lover, + Sighing like furnace, with a woeful ballad + Made to his mistress' eyebrow. Then a soldier, + Full of strange oaths and bearded like the pard, + Jealous in honour, sudden and quick in quarrel, + Seeking the bubble reputation + Even in the cannon's mouth. And then the justice, + In fair round belly with good capon lined, + With eyes severe and beard of formal cut, + Full of wise saws and modern instances; + And so he plays his part. The sixth age shifts + Into the lean and slipper'd pantaloon, + With spectacles on nose and pouch on side, + His youthful hose, well saved, a world too wide + For his shrunk shank; and his big manly voice, + Turning again toward childish treble, pipes + And whistles in his sound. Last scene of all, + That ends this strange eventful history, + Is second childishness and mere oblivion, + Sans teeth, sans eyes, sans taste, sans everything. + + [Re-enter ORLANDO, with ADAM] + +DUKE SENIOR Welcome. Set down your venerable burthen, + And let him feed. + +ORLANDO I thank you most for him. + +ADAM So had you need: + I scarce can speak to thank you for myself. + +DUKE SENIOR Welcome; fall to: I will not trouble you + As yet, to question you about your fortunes. + Give us some music; and, good cousin, sing. + + SONG. +AMIENS Blow, blow, thou winter wind. + Thou art not so unkind + As man's ingratitude; + Thy tooth is not so keen, + Because thou art not seen, + Although thy breath be rude. + Heigh-ho! sing, heigh-ho! unto the green holly: + Most friendship is feigning, most loving mere folly: + Then, heigh-ho, the holly! + This life is most jolly. + Freeze, freeze, thou bitter sky, + That dost not bite so nigh + As benefits forgot: + Though thou the waters warp, + Thy sting is not so sharp + As friend remember'd not. + Heigh-ho! sing, &c. + +DUKE SENIOR If that you were the good Sir Rowland's son, + As you have whisper'd faithfully you were, + And as mine eye doth his effigies witness + Most truly limn'd and living in your face, + Be truly welcome hither: I am the duke + That loved your father: the residue of your fortune, + Go to my cave and tell me. Good old man, + Thou art right welcome as thy master is. + Support him by the arm. Give me your hand, + And let me all your fortunes understand. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT III + + + +SCENE I A room in the palace. + + + [Enter DUKE FREDERICK, Lords, and OLIVER] + +DUKE FREDERICK Not see him since? Sir, sir, that cannot be: + But were I not the better part made mercy, + I should not seek an absent argument + Of my revenge, thou present. But look to it: + Find out thy brother, wheresoe'er he is; + Seek him with candle; bring him dead or living + Within this twelvemonth, or turn thou no more + To seek a living in our territory. + Thy lands and all things that thou dost call thine + Worth seizure do we seize into our hands, + Till thou canst quit thee by thy brothers mouth + Of what we think against thee. + +OLIVER O that your highness knew my heart in this! + I never loved my brother in my life. + +DUKE FREDERICK More villain thou. Well, push him out of doors; + And let my officers of such a nature + Make an extent upon his house and lands: + Do this expediently and turn him going. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT III + + + +SCENE II The forest. + + + [Enter ORLANDO, with a paper] + +ORLANDO Hang there, my verse, in witness of my love: + And thou, thrice-crowned queen of night, survey + With thy chaste eye, from thy pale sphere above, + Thy huntress' name that my full life doth sway. + O Rosalind! these trees shall be my books + And in their barks my thoughts I'll character; + That every eye which in this forest looks + Shall see thy virtue witness'd every where. + Run, run, Orlando; carve on every tree + The fair, the chaste and unexpressive she. + + [Exit] + + [Enter CORIN and TOUCHSTONE] + +CORIN And how like you this shepherd's life, Master Touchstone? + +TOUCHSTONE Truly, shepherd, in respect of itself, it is a good + life, but in respect that it is a shepherd's life, + it is naught. In respect that it is solitary, I + like it very well; but in respect that it is + private, it is a very vile life. Now, in respect it + is in the fields, it pleaseth me well; but in + respect it is not in the court, it is tedious. As + is it a spare life, look you, it fits my humour well; + but as there is no more plenty in it, it goes much + against my stomach. Hast any philosophy in thee, shepherd? + +CORIN No more but that I know the more one sickens the + worse at ease he is; and that he that wants money, + means and content is without three good friends; + that the property of rain is to wet and fire to + burn; that good pasture makes fat sheep, and that a + great cause of the night is lack of the sun; that + he that hath learned no wit by nature nor art may + complain of good breeding or comes of a very dull kindred. + +TOUCHSTONE Such a one is a natural philosopher. Wast ever in + court, shepherd? + +CORIN No, truly. + +TOUCHSTONE Then thou art damned. + +CORIN Nay, I hope. + +TOUCHSTONE Truly, thou art damned like an ill-roasted egg, all + on one side. + +CORIN For not being at court? Your reason. + +TOUCHSTONE Why, if thou never wast at court, thou never sawest + good manners; if thou never sawest good manners, + then thy manners must be wicked; and wickedness is + sin, and sin is damnation. Thou art in a parlous + state, shepherd. + +CORIN Not a whit, Touchstone: those that are good manners + at the court are as ridiculous in the country as the + behavior of the country is most mockable at the + court. You told me you salute not at the court, but + you kiss your hands: that courtesy would be + uncleanly, if courtiers were shepherds. + +TOUCHSTONE Instance, briefly; come, instance. + +CORIN Why, we are still handling our ewes, and their + fells, you know, are greasy. + +TOUCHSTONE Why, do not your courtier's hands sweat? and is not + the grease of a mutton as wholesome as the sweat of + a man? Shallow, shallow. A better instance, I say; come. + +CORIN Besides, our hands are hard. + +TOUCHSTONE Your lips will feel them the sooner. Shallow again. + A more sounder instance, come. + +CORIN And they are often tarred over with the surgery of + our sheep: and would you have us kiss tar? The + courtier's hands are perfumed with civet. + +TOUCHSTONE Most shallow man! thou worms-meat, in respect of a + good piece of flesh indeed! Learn of the wise, and + perpend: civet is of a baser birth than tar, the + very uncleanly flux of a cat. Mend the instance, shepherd. + +CORIN You have too courtly a wit for me: I'll rest. + +TOUCHSTONE Wilt thou rest damned? God help thee, shallow man! + God make incision in thee! thou art raw. + +CORIN Sir, I am a true labourer: I earn that I eat, get + that I wear, owe no man hate, envy no man's + happiness, glad of other men's good, content with my + harm, and the greatest of my pride is to see my ewes + graze and my lambs suck. + +TOUCHSTONE That is another simple sin in you, to bring the ewes + and the rams together and to offer to get your + living by the copulation of cattle; to be bawd to a + bell-wether, and to betray a she-lamb of a + twelvemonth to a crooked-pated, old, cuckoldly ram, + out of all reasonable match. If thou beest not + damned for this, the devil himself will have no + shepherds; I cannot see else how thou shouldst + 'scape. + +CORIN Here comes young Master Ganymede, my new mistress's brother. + + [Enter ROSALIND, with a paper, reading] + +ROSALIND From the east to western Ind, + No jewel is like Rosalind. + Her worth, being mounted on the wind, + Through all the world bears Rosalind. + All the pictures fairest lined + Are but black to Rosalind. + Let no fair be kept in mind + But the fair of Rosalind. + +TOUCHSTONE I'll rhyme you so eight years together, dinners and + suppers and sleeping-hours excepted: it is the + right butter-women's rank to market. + +ROSALIND Out, fool! + +TOUCHSTONE For a taste: + If a hart do lack a hind, + Let him seek out Rosalind. + If the cat will after kind, + So be sure will Rosalind. + Winter garments must be lined, + So must slender Rosalind. + They that reap must sheaf and bind; + Then to cart with Rosalind. + Sweetest nut hath sourest rind, + Such a nut is Rosalind. + He that sweetest rose will find + Must find love's prick and Rosalind. + This is the very false gallop of verses: why do you + infect yourself with them? + +ROSALIND Peace, you dull fool! I found them on a tree. + +TOUCHSTONE Truly, the tree yields bad fruit. + +ROSALIND I'll graff it with you, and then I shall graff it + with a medlar: then it will be the earliest fruit + i' the country; for you'll be rotten ere you be half + ripe, and that's the right virtue of the medlar. + +TOUCHSTONE You have said; but whether wisely or no, let the + forest judge. + + [Enter CELIA, with a writing] + +ROSALIND Peace! Here comes my sister, reading: stand aside. + +CELIA [Reads] + + Why should this a desert be? + For it is unpeopled? No: + Tongues I'll hang on every tree, + That shall civil sayings show: + Some, how brief the life of man + Runs his erring pilgrimage, + That the stretching of a span + Buckles in his sum of age; + Some, of violated vows + 'Twixt the souls of friend and friend: + But upon the fairest boughs, + Or at every sentence end, + Will I Rosalinda write, + Teaching all that read to know + The quintessence of every sprite + Heaven would in little show. + Therefore Heaven Nature charged + That one body should be fill'd + With all graces wide-enlarged: + Nature presently distill'd + Helen's cheek, but not her heart, + Cleopatra's majesty, + Atalanta's better part, + Sad Lucretia's modesty. + Thus Rosalind of many parts + By heavenly synod was devised, + Of many faces, eyes and hearts, + To have the touches dearest prized. + Heaven would that she these gifts should have, + And I to live and die her slave. + +ROSALIND O most gentle pulpiter! what tedious homily of love + have you wearied your parishioners withal, and never + cried 'Have patience, good people!' + +CELIA How now! back, friends! Shepherd, go off a little. + Go with him, sirrah. + +TOUCHSTONE Come, shepherd, let us make an honourable retreat; + though not with bag and baggage, yet with scrip and scrippage. + + [Exeunt CORIN and TOUCHSTONE] + +CELIA Didst thou hear these verses? + +ROSALIND O, yes, I heard them all, and more too; for some of + them had in them more feet than the verses would bear. + +CELIA That's no matter: the feet might bear the verses. + +ROSALIND Ay, but the feet were lame and could not bear + themselves without the verse and therefore stood + lamely in the verse. + +CELIA But didst thou hear without wondering how thy name + should be hanged and carved upon these trees? + +ROSALIND I was seven of the nine days out of the wonder + before you came; for look here what I found on a + palm-tree. I was never so be-rhymed since + Pythagoras' time, that I was an Irish rat, which I + can hardly remember. + +CELIA Trow you who hath done this? + +ROSALIND Is it a man? + +CELIA And a chain, that you once wore, about his neck. + Change you colour? + +ROSALIND I prithee, who? + +CELIA O Lord, Lord! it is a hard matter for friends to + meet; but mountains may be removed with earthquakes + and so encounter. + +ROSALIND Nay, but who is it? + +CELIA Is it possible? + +ROSALIND Nay, I prithee now with most petitionary vehemence, + tell me who it is. + +CELIA O wonderful, wonderful, and most wonderful + wonderful! and yet again wonderful, and after that, + out of all hooping! + +ROSALIND Good my complexion! dost thou think, though I am + caparisoned like a man, I have a doublet and hose in + my disposition? One inch of delay more is a + South-sea of discovery; I prithee, tell me who is it + quickly, and speak apace. I would thou couldst + stammer, that thou mightst pour this concealed man + out of thy mouth, as wine comes out of a narrow- + mouthed bottle, either too much at once, or none at + all. I prithee, take the cork out of thy mouth that + may drink thy tidings. + +CELIA So you may put a man in your belly. + +ROSALIND Is he of God's making? What manner of man? Is his + head worth a hat, or his chin worth a beard? + +CELIA Nay, he hath but a little beard. + +ROSALIND Why, God will send more, if the man will be + thankful: let me stay the growth of his beard, if + thou delay me not the knowledge of his chin. + +CELIA It is young Orlando, that tripped up the wrestler's + heels and your heart both in an instant. + +ROSALIND Nay, but the devil take mocking: speak, sad brow and + true maid. + +CELIA I' faith, coz, 'tis he. + +ROSALIND Orlando? + +CELIA Orlando. + +ROSALIND Alas the day! what shall I do with my doublet and + hose? What did he when thou sawest him? What said + he? How looked he? Wherein went he? What makes + him here? Did he ask for me? Where remains he? + How parted he with thee? and when shalt thou see + him again? Answer me in one word. + +CELIA You must borrow me Gargantua's mouth first: 'tis a + word too great for any mouth of this age's size. To + say ay and no to these particulars is more than to + answer in a catechism. + +ROSALIND But doth he know that I am in this forest and in + man's apparel? Looks he as freshly as he did the + day he wrestled? + +CELIA It is as easy to count atomies as to resolve the + propositions of a lover; but take a taste of my + finding him, and relish it with good observance. + I found him under a tree, like a dropped acorn. + +ROSALIND It may well be called Jove's tree, when it drops + forth such fruit. + +CELIA Give me audience, good madam. + +ROSALIND Proceed. + +CELIA There lay he, stretched along, like a wounded knight. + +ROSALIND Though it be pity to see such a sight, it well + becomes the ground. + +CELIA Cry 'holla' to thy tongue, I prithee; it curvets + unseasonably. He was furnished like a hunter. + +ROSALIND O, ominous! he comes to kill my heart. + +CELIA I would sing my song without a burden: thou bringest + me out of tune. + +ROSALIND Do you not know I am a woman? when I think, I must + speak. Sweet, say on. + +CELIA You bring me out. Soft! comes he not here? + + [Enter ORLANDO and JAQUES] + +ROSALIND 'Tis he: slink by, and note him. + +JAQUES I thank you for your company; but, good faith, I had + as lief have been myself alone. + +ORLANDO And so had I; but yet, for fashion sake, I thank you + too for your society. + +JAQUES God be wi' you: let's meet as little as we can. + +ORLANDO I do desire we may be better strangers. + +JAQUES I pray you, mar no more trees with writing + love-songs in their barks. + +ORLANDO I pray you, mar no more of my verses with reading + them ill-favouredly. + +JAQUES Rosalind is your love's name? + +ORLANDO Yes, just. + +JAQUES I do not like her name. + +ORLANDO There was no thought of pleasing you when she was + christened. + +JAQUES What stature is she of? + +ORLANDO Just as high as my heart. + +JAQUES You are full of pretty answers. Have you not been + acquainted with goldsmiths' wives, and conned them + out of rings? + +ORLANDO Not so; but I answer you right painted cloth, from + whence you have studied your questions. + +JAQUES You have a nimble wit: I think 'twas made of + Atalanta's heels. Will you sit down with me? and + we two will rail against our mistress the world and + all our misery. + +ORLANDO I will chide no breather in the world but myself, + against whom I know most faults. + +JAQUES The worst fault you have is to be in love. + +ORLANDO 'Tis a fault I will not change for your best virtue. + I am weary of you. + +JAQUES By my troth, I was seeking for a fool when I found + you. + +ORLANDO He is drowned in the brook: look but in, and you + shall see him. + +JAQUES There I shall see mine own figure. + +ORLANDO Which I take to be either a fool or a cipher. + +JAQUES I'll tarry no longer with you: farewell, good + Signior Love. + +ORLANDO I am glad of your departure: adieu, good Monsieur + Melancholy. + + [Exit JAQUES] + +ROSALIND [Aside to CELIA] I will speak to him, like a saucy + lackey and under that habit play the knave with him. + Do you hear, forester? + +ORLANDO Very well: what would you? + +ROSALIND I pray you, what is't o'clock? + +ORLANDO You should ask me what time o' day: there's no clock + in the forest. + +ROSALIND Then there is no true lover in the forest; else + sighing every minute and groaning every hour would + detect the lazy foot of Time as well as a clock. + +ORLANDO And why not the swift foot of Time? had not that + been as proper? + +ROSALIND By no means, sir: Time travels in divers paces with + divers persons. I'll tell you who Time ambles + withal, who Time trots withal, who Time gallops + withal and who he stands still withal. + +ORLANDO I prithee, who doth he trot withal? + +ROSALIND Marry, he trots hard with a young maid between the + contract of her marriage and the day it is + solemnized: if the interim be but a se'nnight, + Time's pace is so hard that it seems the length of + seven year. + +ORLANDO Who ambles Time withal? + +ROSALIND With a priest that lacks Latin and a rich man that + hath not the gout, for the one sleeps easily because + he cannot study, and the other lives merrily because + he feels no pain, the one lacking the burden of lean + and wasteful learning, the other knowing no burden + of heavy tedious penury; these Time ambles withal. + +ORLANDO Who doth he gallop withal? + +ROSALIND With a thief to the gallows, for though he go as + softly as foot can fall, he thinks himself too soon there. + +ORLANDO Who stays it still withal? + +ROSALIND With lawyers in the vacation, for they sleep between + term and term and then they perceive not how Time moves. + +ORLANDO Where dwell you, pretty youth? + +ROSALIND With this shepherdess, my sister; here in the + skirts of the forest, like fringe upon a petticoat. + +ORLANDO Are you native of this place? + +ROSALIND As the cony that you see dwell where she is kindled. + +ORLANDO Your accent is something finer than you could + purchase in so removed a dwelling. + +ROSALIND I have been told so of many: but indeed an old + religious uncle of mine taught me to speak, who was + in his youth an inland man; one that knew courtship + too well, for there he fell in love. I have heard + him read many lectures against it, and I thank God + I am not a woman, to be touched with so many + giddy offences as he hath generally taxed their + whole sex withal. + +ORLANDO Can you remember any of the principal evils that he + laid to the charge of women? + +ROSALIND There were none principal; they were all like one + another as half-pence are, every one fault seeming + monstrous till his fellow fault came to match it. + +ORLANDO I prithee, recount some of them. + +ROSALIND No, I will not cast away my physic but on those that + are sick. There is a man haunts the forest, that + abuses our young plants with carving 'Rosalind' on + their barks; hangs odes upon hawthorns and elegies + on brambles, all, forsooth, deifying the name of + Rosalind: if I could meet that fancy-monger I would + give him some good counsel, for he seems to have the + quotidian of love upon him. + +ORLANDO I am he that is so love-shaked: I pray you tell me + your remedy. + +ROSALIND There is none of my uncle's marks upon you: he + taught me how to know a man in love; in which cage + of rushes I am sure you are not prisoner. + +ORLANDO What were his marks? + +ROSALIND A lean cheek, which you have not, a blue eye and + sunken, which you have not, an unquestionable + spirit, which you have not, a beard neglected, + which you have not; but I pardon you for that, for + simply your having in beard is a younger brother's + revenue: then your hose should be ungartered, your + bonnet unbanded, your sleeve unbuttoned, your shoe + untied and every thing about you demonstrating a + careless desolation; but you are no such man; you + are rather point-device in your accoutrements as + loving yourself than seeming the lover of any other. + +ORLANDO Fair youth, I would I could make thee believe I love. + +ROSALIND Me believe it! you may as soon make her that you + love believe it; which, I warrant, she is apter to + do than to confess she does: that is one of the + points in the which women still give the lie to + their consciences. But, in good sooth, are you he + that hangs the verses on the trees, wherein Rosalind + is so admired? + +ORLANDO I swear to thee, youth, by the white hand of + Rosalind, I am that he, that unfortunate he. + +ROSALIND But are you so much in love as your rhymes speak? + +ORLANDO Neither rhyme nor reason can express how much. + +ROSALIND Love is merely a madness, and, I tell you, deserves + as well a dark house and a whip as madmen do: and + the reason why they are not so punished and cured + is, that the lunacy is so ordinary that the whippers + are in love too. Yet I profess curing it by counsel. + +ORLANDO Did you ever cure any so? + +ROSALIND Yes, one, and in this manner. He was to imagine me + his love, his mistress; and I set him every day to + woo me: at which time would I, being but a moonish + youth, grieve, be effeminate, changeable, longing + and liking, proud, fantastical, apish, shallow, + inconstant, full of tears, full of smiles, for every + passion something and for no passion truly any + thing, as boys and women are for the most part + cattle of this colour; would now like him, now loathe + him; then entertain him, then forswear him; now weep + for him, then spit at him; that I drave my suitor + from his mad humour of love to a living humour of + madness; which was, to forswear the full stream of + the world, and to live in a nook merely monastic. + And thus I cured him; and this way will I take upon + me to wash your liver as clean as a sound sheep's + heart, that there shall not be one spot of love in't. + +ORLANDO I would not be cured, youth. + +ROSALIND I would cure you, if you would but call me Rosalind + and come every day to my cote and woo me. + +ORLANDO Now, by the faith of my love, I will: tell me + where it is. + +ROSALIND Go with me to it and I'll show it you and by the way + you shall tell me where in the forest you live. + Will you go? + +ORLANDO With all my heart, good youth. + +ROSALIND Nay you must call me Rosalind. Come, sister, will you go? + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT III + + + +SCENE III The forest. + + + [Enter TOUCHSTONE and AUDREY; JAQUES behind] + +TOUCHSTONE Come apace, good Audrey: I will fetch up your + goats, Audrey. And how, Audrey? am I the man yet? + doth my simple feature content you? + +AUDREY Your features! Lord warrant us! what features! + +TOUCHSTONE I am here with thee and thy goats, as the most + capricious poet, honest Ovid, was among the Goths. + +JAQUES [Aside] O knowledge ill-inhabited, worse than Jove + in a thatched house! + +TOUCHSTONE When a man's verses cannot be understood, nor a + man's good wit seconded with the forward child + Understanding, it strikes a man more dead than a + great reckoning in a little room. Truly, I would + the gods had made thee poetical. + +AUDREY I do not know what 'poetical' is: is it honest in + deed and word? is it a true thing? + +TOUCHSTONE No, truly; for the truest poetry is the most + feigning; and lovers are given to poetry, and what + they swear in poetry may be said as lovers they do feign. + +AUDREY Do you wish then that the gods had made me poetical? + +TOUCHSTONE I do, truly; for thou swearest to me thou art + honest: now, if thou wert a poet, I might have some + hope thou didst feign. + +AUDREY Would you not have me honest? + +TOUCHSTONE No, truly, unless thou wert hard-favoured; for + honesty coupled to beauty is to have honey a sauce to sugar. + +JAQUES [Aside] A material fool! + +AUDREY Well, I am not fair; and therefore I pray the gods + make me honest. + +TOUCHSTONE Truly, and to cast away honesty upon a foul slut + were to put good meat into an unclean dish. + +AUDREY I am not a slut, though I thank the gods I am foul. + +TOUCHSTONE Well, praised be the gods for thy foulness! + sluttishness may come hereafter. But be it as it may + be, I will marry thee, and to that end I have been + with Sir Oliver Martext, the vicar of the next + village, who hath promised to meet me in this place + of the forest and to couple us. + +JAQUES [Aside] I would fain see this meeting. + +AUDREY Well, the gods give us joy! + +TOUCHSTONE Amen. A man may, if he were of a fearful heart, + stagger in this attempt; for here we have no temple + but the wood, no assembly but horn-beasts. But what + though? Courage! As horns are odious, they are + necessary. It is said, 'many a man knows no end of + his goods:' right; many a man has good horns, and + knows no end of them. Well, that is the dowry of + his wife; 'tis none of his own getting. Horns? + Even so. Poor men alone? No, no; the noblest deer + hath them as huge as the rascal. Is the single man + therefore blessed? No: as a walled town is more + worthier than a village, so is the forehead of a + married man more honourable than the bare brow of a + bachelor; and by how much defence is better than no + skill, by so much is a horn more precious than to + want. Here comes Sir Oliver. + + [Enter SIR OLIVER MARTEXT] + + Sir Oliver Martext, you are well met: will you + dispatch us here under this tree, or shall we go + with you to your chapel? + +SIR OLIVER MARTEXT Is there none here to give the woman? + +TOUCHSTONE I will not take her on gift of any man. + +SIR OLIVER MARTEXT Truly, she must be given, or the marriage is not lawful. + +JAQUES [Advancing] + + Proceed, proceed I'll give her. + +TOUCHSTONE Good even, good Master What-ye-call't: how do you, + sir? You are very well met: God 'ild you for your + last company: I am very glad to see you: even a + toy in hand here, sir: nay, pray be covered. + +JAQUES Will you be married, motley? + +TOUCHSTONE As the ox hath his bow, sir, the horse his curb and + the falcon her bells, so man hath his desires; and + as pigeons bill, so wedlock would be nibbling. + +JAQUES And will you, being a man of your breeding, be + married under a bush like a beggar? Get you to + church, and have a good priest that can tell you + what marriage is: this fellow will but join you + together as they join wainscot; then one of you will + prove a shrunk panel and, like green timber, warp, warp. + +TOUCHSTONE [Aside] I am not in the mind but I were better to be + married of him than of another: for he is not like + to marry me well; and not being well married, it + will be a good excuse for me hereafter to leave my wife. + +JAQUES Go thou with me, and let me counsel thee. + +TOUCHSTONE 'Come, sweet Audrey: + We must be married, or we must live in bawdry. + Farewell, good Master Oliver: not,-- + O sweet Oliver, + O brave Oliver, + Leave me not behind thee: but,-- + Wind away, + Begone, I say, + I will not to wedding with thee. + + [Exeunt JAQUES, TOUCHSTONE and AUDREY] + +SIR OLIVER MARTEXT 'Tis no matter: ne'er a fantastical knave of them + all shall flout me out of my calling. + + [Exit] + + + + + AS YOU LIKE IT + + +ACT III + + + +SCENE IV The forest. + + + [Enter ROSALIND and CELIA] + +ROSALIND Never talk to me; I will weep. + +CELIA Do, I prithee; but yet have the grace to consider + that tears do not become a man. + +ROSALIND But have I not cause to weep? + +CELIA As good cause as one would desire; therefore weep. + +ROSALIND His very hair is of the dissembling colour. + +CELIA Something browner than Judas's marry, his kisses are + Judas's own children. + +ROSALIND I' faith, his hair is of a good colour. + +CELIA An excellent colour: your chestnut was ever the only colour. + +ROSALIND And his kissing is as full of sanctity as the touch + of holy bread. + +CELIA He hath bought a pair of cast lips of Diana: a nun + of winter's sisterhood kisses not more religiously; + the very ice of chastity is in them. + +ROSALIND But why did he swear he would come this morning, and + comes not? + +CELIA Nay, certainly, there is no truth in him. + +ROSALIND Do you think so? + +CELIA Yes; I think he is not a pick-purse nor a + horse-stealer, but for his verity in love, I do + think him as concave as a covered goblet or a + worm-eaten nut. + +ROSALIND Not true in love? + +CELIA Yes, when he is in; but I think he is not in. + +ROSALIND You have heard him swear downright he was. + +CELIA 'Was' is not 'is:' besides, the oath of a lover is + no stronger than the word of a tapster; they are + both the confirmer of false reckonings. He attends + here in the forest on the duke your father. + +ROSALIND I met the duke yesterday and had much question with + him: he asked me of what parentage I was; I told + him, of as good as he; so he laughed and let me go. + But what talk we of fathers, when there is such a + man as Orlando? + +CELIA O, that's a brave man! he writes brave verses, + speaks brave words, swears brave oaths and breaks + them bravely, quite traverse, athwart the heart of + his lover; as a puisny tilter, that spurs his horse + but on one side, breaks his staff like a noble + goose: but all's brave that youth mounts and folly + guides. Who comes here? + + [Enter CORIN] + +CORIN Mistress and master, you have oft inquired + After the shepherd that complain'd of love, + Who you saw sitting by me on the turf, + Praising the proud disdainful shepherdess + That was his mistress. + +CELIA Well, and what of him? + +CORIN If you will see a pageant truly play'd, + Between the pale complexion of true love + And the red glow of scorn and proud disdain, + Go hence a little and I shall conduct you, + If you will mark it. + +ROSALIND O, come, let us remove: + The sight of lovers feedeth those in love. + Bring us to this sight, and you shall say + I'll prove a busy actor in their play. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT III + + + +SCENE V Another part of the forest. + + + [Enter SILVIUS and PHEBE] + +SILVIUS Sweet Phebe, do not scorn me; do not, Phebe; + Say that you love me not, but say not so + In bitterness. The common executioner, + Whose heart the accustom'd sight of death makes hard, + Falls not the axe upon the humbled neck + But first begs pardon: will you sterner be + Than he that dies and lives by bloody drops? + + [Enter ROSALIND, CELIA, and CORIN, behind] + +PHEBE I would not be thy executioner: + I fly thee, for I would not injure thee. + Thou tell'st me there is murder in mine eye: + 'Tis pretty, sure, and very probable, + That eyes, that are the frail'st and softest things, + Who shut their coward gates on atomies, + Should be call'd tyrants, butchers, murderers! + Now I do frown on thee with all my heart; + And if mine eyes can wound, now let them kill thee: + Now counterfeit to swoon; why now fall down; + Or if thou canst not, O, for shame, for shame, + Lie not, to say mine eyes are murderers! + Now show the wound mine eye hath made in thee: + Scratch thee but with a pin, and there remains + Some scar of it; lean but upon a rush, + The cicatrice and capable impressure + Thy palm some moment keeps; but now mine eyes, + Which I have darted at thee, hurt thee not, + Nor, I am sure, there is no force in eyes + That can do hurt. + +SILVIUS O dear Phebe, + If ever,--as that ever may be near,-- + You meet in some fresh cheek the power of fancy, + Then shall you know the wounds invisible + That love's keen arrows make. + +PHEBE But till that time + Come not thou near me: and when that time comes, + Afflict me with thy mocks, pity me not; + As till that time I shall not pity thee. + +ROSALIND And why, I pray you? Who might be your mother, + That you insult, exult, and all at once, + Over the wretched? What though you have no beauty,-- + As, by my faith, I see no more in you + Than without candle may go dark to bed-- + Must you be therefore proud and pitiless? + Why, what means this? Why do you look on me? + I see no more in you than in the ordinary + Of nature's sale-work. 'Od's my little life, + I think she means to tangle my eyes too! + No, faith, proud mistress, hope not after it: + 'Tis not your inky brows, your black silk hair, + Your bugle eyeballs, nor your cheek of cream, + That can entame my spirits to your worship. + You foolish shepherd, wherefore do you follow her, + Like foggy south puffing with wind and rain? + You are a thousand times a properer man + Than she a woman: 'tis such fools as you + That makes the world full of ill-favour'd children: + 'Tis not her glass, but you, that flatters her; + And out of you she sees herself more proper + Than any of her lineaments can show her. + But, mistress, know yourself: down on your knees, + And thank heaven, fasting, for a good man's love: + For I must tell you friendly in your ear, + Sell when you can: you are not for all markets: + Cry the man mercy; love him; take his offer: + Foul is most foul, being foul to be a scoffer. + So take her to thee, shepherd: fare you well. + +PHEBE Sweet youth, I pray you, chide a year together: + I had rather hear you chide than this man woo. + +ROSALIND He's fallen in love with your foulness and she'll + fall in love with my anger. If it be so, as fast as + she answers thee with frowning looks, I'll sauce her + with bitter words. Why look you so upon me? + +PHEBE For no ill will I bear you. + +ROSALIND I pray you, do not fall in love with me, + For I am falser than vows made in wine: + Besides, I like you not. If you will know my house, + 'Tis at the tuft of olives here hard by. + Will you go, sister? Shepherd, ply her hard. + Come, sister. Shepherdess, look on him better, + And be not proud: though all the world could see, + None could be so abused in sight as he. + Come, to our flock. + + [Exeunt ROSALIND, CELIA and CORIN] + +PHEBE Dead Shepherd, now I find thy saw of might, + 'Who ever loved that loved not at first sight?' + +SILVIUS Sweet Phebe,-- + +PHEBE Ha, what say'st thou, Silvius? + +SILVIUS Sweet Phebe, pity me. + +PHEBE Why, I am sorry for thee, gentle Silvius. + +SILVIUS Wherever sorrow is, relief would be: + If you do sorrow at my grief in love, + By giving love your sorrow and my grief + Were both extermined. + +PHEBE Thou hast my love: is not that neighbourly? + +SILVIUS I would have you. + +PHEBE Why, that were covetousness. + Silvius, the time was that I hated thee, + And yet it is not that I bear thee love; + But since that thou canst talk of love so well, + Thy company, which erst was irksome to me, + I will endure, and I'll employ thee too: + But do not look for further recompense + Than thine own gladness that thou art employ'd. + +SILVIUS So holy and so perfect is my love, + And I in such a poverty of grace, + That I shall think it a most plenteous crop + To glean the broken ears after the man + That the main harvest reaps: loose now and then + A scatter'd smile, and that I'll live upon. + +PHEBE Know'st now the youth that spoke to me erewhile? + +SILVIUS Not very well, but I have met him oft; + And he hath bought the cottage and the bounds + That the old carlot once was master of. + +PHEBE Think not I love him, though I ask for him: + 'Tis but a peevish boy; yet he talks well; + But what care I for words? yet words do well + When he that speaks them pleases those that hear. + It is a pretty youth: not very pretty: + But, sure, he's proud, and yet his pride becomes him: + He'll make a proper man: the best thing in him + Is his complexion; and faster than his tongue + Did make offence his eye did heal it up. + He is not very tall; yet for his years he's tall: + His leg is but so so; and yet 'tis well: + There was a pretty redness in his lip, + A little riper and more lusty red + Than that mix'd in his cheek; 'twas just the difference + Between the constant red and mingled damask. + There be some women, Silvius, had they mark'd him + In parcels as I did, would have gone near + To fall in love with him; but, for my part, + I love him not nor hate him not; and yet + I have more cause to hate him than to love him: + For what had he to do to chide at me? + He said mine eyes were black and my hair black: + And, now I am remember'd, scorn'd at me: + I marvel why I answer'd not again: + But that's all one; omittance is no quittance. + I'll write to him a very taunting letter, + And thou shalt bear it: wilt thou, Silvius? + +SILVIUS Phebe, with all my heart. + +PHEBE I'll write it straight; + The matter's in my head and in my heart: + I will be bitter with him and passing short. + Go with me, Silvius. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT IV + + + +SCENE I The forest. + + + [Enter ROSALIND, CELIA, and JAQUES] + +JAQUES I prithee, pretty youth, let me be better acquainted + with thee. + +ROSALIND They say you are a melancholy fellow. + +JAQUES I am so; I do love it better than laughing. + +ROSALIND Those that are in extremity of either are abominable + fellows and betray themselves to every modern + censure worse than drunkards. + +JAQUES Why, 'tis good to be sad and say nothing. + +ROSALIND Why then, 'tis good to be a post. + +JAQUES I have neither the scholar's melancholy, which is + emulation, nor the musician's, which is fantastical, + nor the courtier's, which is proud, nor the + soldier's, which is ambitious, nor the lawyer's, + which is politic, nor the lady's, which is nice, nor + the lover's, which is all these: but it is a + melancholy of mine own, compounded of many simples, + extracted from many objects, and indeed the sundry's + contemplation of my travels, in which my often + rumination wraps me m a most humorous sadness. + +ROSALIND A traveller! By my faith, you have great reason to + be sad: I fear you have sold your own lands to see + other men's; then, to have seen much and to have + nothing, is to have rich eyes and poor hands. + +JAQUES Yes, I have gained my experience. + +ROSALIND And your experience makes you sad: I had rather have + a fool to make me merry than experience to make me + sad; and to travel for it too! + + [Enter ORLANDO] + +ORLANDO Good day and happiness, dear Rosalind! + +JAQUES Nay, then, God be wi' you, an you talk in blank verse. + + [Exit] + +ROSALIND Farewell, Monsieur Traveller: look you lisp and + wear strange suits, disable all the benefits of your + own country, be out of love with your nativity and + almost chide God for making you that countenance you + are, or I will scarce think you have swam in a + gondola. Why, how now, Orlando! where have you been + all this while? You a lover! An you serve me such + another trick, never come in my sight more. + +ORLANDO My fair Rosalind, I come within an hour of my promise. + +ROSALIND Break an hour's promise in love! He that will + divide a minute into a thousand parts and break but + a part of the thousandth part of a minute in the + affairs of love, it may be said of him that Cupid + hath clapped him o' the shoulder, but I'll warrant + him heart-whole. + +ORLANDO Pardon me, dear Rosalind. + +ROSALIND Nay, an you be so tardy, come no more in my sight: I + had as lief be wooed of a snail. + +ORLANDO Of a snail? + +ROSALIND Ay, of a snail; for though he comes slowly, he + carries his house on his head; a better jointure, + I think, than you make a woman: besides he brings + his destiny with him. + +ORLANDO What's that? + +ROSALIND Why, horns, which such as you are fain to be + beholding to your wives for: but he comes armed in + his fortune and prevents the slander of his wife. + +ORLANDO Virtue is no horn-maker; and my Rosalind is virtuous. + +ROSALIND And I am your Rosalind. + +CELIA It pleases him to call you so; but he hath a + Rosalind of a better leer than you. + +ROSALIND Come, woo me, woo me, for now I am in a holiday + humour and like enough to consent. What would you + say to me now, an I were your very very Rosalind? + +ORLANDO I would kiss before I spoke. + +ROSALIND Nay, you were better speak first, and when you were + gravelled for lack of matter, you might take + occasion to kiss. Very good orators, when they are + out, they will spit; and for lovers lacking--God + warn us!--matter, the cleanliest shift is to kiss. + +ORLANDO How if the kiss be denied? + +ROSALIND Then she puts you to entreaty, and there begins new matter. + +ORLANDO Who could be out, being before his beloved mistress? + +ROSALIND Marry, that should you, if I were your mistress, or + I should think my honesty ranker than my wit. + +ORLANDO What, of my suit? + +ROSALIND Not out of your apparel, and yet out of your suit. + Am not I your Rosalind? + +ORLANDO I take some joy to say you are, because I would be + talking of her. + +ROSALIND Well in her person I say I will not have you. + +ORLANDO Then in mine own person I die. + +ROSALIND No, faith, die by attorney. The poor world is + almost six thousand years old, and in all this time + there was not any man died in his own person, + videlicit, in a love-cause. Troilus had his brains + dashed out with a Grecian club; yet he did what he + could to die before, and he is one of the patterns + of love. Leander, he would have lived many a fair + year, though Hero had turned nun, if it had not been + for a hot midsummer night; for, good youth, he went + but forth to wash him in the Hellespont and being + taken with the cramp was drowned and the foolish + coroners of that age found it was 'Hero of Sestos.' + But these are all lies: men have died from time to + time and worms have eaten them, but not for love. + +ORLANDO I would not have my right Rosalind of this mind, + for, I protest, her frown might kill me. + +ROSALIND By this hand, it will not kill a fly. But come, now + I will be your Rosalind in a more coming-on + disposition, and ask me what you will. I will grant + it. + +ORLANDO Then love me, Rosalind. + +ROSALIND Yes, faith, will I, Fridays and Saturdays and all. + +ORLANDO And wilt thou have me? + +ROSALIND Ay, and twenty such. + +ORLANDO What sayest thou? + +ROSALIND Are you not good? + +ORLANDO I hope so. + +ROSALIND Why then, can one desire too much of a good thing? + Come, sister, you shall be the priest and marry us. + Give me your hand, Orlando. What do you say, sister? + +ORLANDO Pray thee, marry us. + +CELIA I cannot say the words. + +ROSALIND You must begin, 'Will you, Orlando--' + +CELIA Go to. Will you, Orlando, have to wife this Rosalind? + +ORLANDO I will. + +ROSALIND Ay, but when? + +ORLANDO Why now; as fast as she can marry us. + +ROSALIND Then you must say 'I take thee, Rosalind, for wife.' + +ORLANDO I take thee, Rosalind, for wife. + +ROSALIND I might ask you for your commission; but I do take + thee, Orlando, for my husband: there's a girl goes + before the priest; and certainly a woman's thought + runs before her actions. + +ORLANDO So do all thoughts; they are winged. + +ROSALIND Now tell me how long you would have her after you + have possessed her. + +ORLANDO For ever and a day. + +ROSALIND Say 'a day,' without the 'ever.' No, no, Orlando; + men are April when they woo, December when they wed: + maids are May when they are maids, but the sky + changes when they are wives. I will be more jealous + of thee than a Barbary cock-pigeon over his hen, + more clamorous than a parrot against rain, more + new-fangled than an ape, more giddy in my desires + than a monkey: I will weep for nothing, like Diana + in the fountain, and I will do that when you are + disposed to be merry; I will laugh like a hyen, and + that when thou art inclined to sleep. + +ORLANDO But will my Rosalind do so? + +ROSALIND By my life, she will do as I do. + +ORLANDO O, but she is wise. + +ROSALIND Or else she could not have the wit to do this: the + wiser, the waywarder: make the doors upon a woman's + wit and it will out at the casement; shut that and + 'twill out at the key-hole; stop that, 'twill fly + with the smoke out at the chimney. + +ORLANDO A man that had a wife with such a wit, he might say + 'Wit, whither wilt?' + +ROSALIND Nay, you might keep that cheque for it till you met + your wife's wit going to your neighbour's bed. + +ORLANDO And what wit could wit have to excuse that? + +ROSALIND Marry, to say she came to seek you there. You shall + never take her without her answer, unless you take + her without her tongue. O, that woman that cannot + make her fault her husband's occasion, let her + never nurse her child herself, for she will breed + it like a fool! + +ORLANDO For these two hours, Rosalind, I will leave thee. + +ROSALIND Alas! dear love, I cannot lack thee two hours. + +ORLANDO I must attend the duke at dinner: by two o'clock I + will be with thee again. + +ROSALIND Ay, go your ways, go your ways; I knew what you + would prove: my friends told me as much, and I + thought no less: that flattering tongue of yours + won me: 'tis but one cast away, and so, come, + death! Two o'clock is your hour? + +ORLANDO Ay, sweet Rosalind. + +ROSALIND By my troth, and in good earnest, and so God mend + me, and by all pretty oaths that are not dangerous, + if you break one jot of your promise or come one + minute behind your hour, I will think you the most + pathetical break-promise and the most hollow lover + and the most unworthy of her you call Rosalind that + may be chosen out of the gross band of the + unfaithful: therefore beware my censure and keep + your promise. + +ORLANDO With no less religion than if thou wert indeed my + Rosalind: so adieu. + +ROSALIND Well, Time is the old justice that examines all such + offenders, and let Time try: adieu. + + [Exit ORLANDO] + +CELIA You have simply misused our sex in your love-prate: + we must have your doublet and hose plucked over your + head, and show the world what the bird hath done to + her own nest. + +ROSALIND O coz, coz, coz, my pretty little coz, that thou + didst know how many fathom deep I am in love! But + it cannot be sounded: my affection hath an unknown + bottom, like the bay of Portugal. + +CELIA Or rather, bottomless, that as fast as you pour + affection in, it runs out. + +ROSALIND No, that same wicked bastard of Venus that was begot + of thought, conceived of spleen and born of madness, + that blind rascally boy that abuses every one's eyes + because his own are out, let him be judge how deep I + am in love. I'll tell thee, Aliena, I cannot be out + of the sight of Orlando: I'll go find a shadow and + sigh till he come. + +CELIA And I'll sleep. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT IV + + + +SCENE II The forest. + + + [Enter JAQUES, Lords, and Foresters] + +JAQUES Which is he that killed the deer? + +A Lord Sir, it was I. + +JAQUES Let's present him to the duke, like a Roman + conqueror; and it would do well to set the deer's + horns upon his head, for a branch of victory. Have + you no song, forester, for this purpose? + +Forester Yes, sir. + +JAQUES Sing it: 'tis no matter how it be in tune, so it + make noise enough. + + SONG. +Forester What shall he have that kill'd the deer? + His leather skin and horns to wear. + Then sing him home; + + [The rest shall bear this burden] + + Take thou no scorn to wear the horn; + It was a crest ere thou wast born: + Thy father's father wore it, + And thy father bore it: + The horn, the horn, the lusty horn + Is not a thing to laugh to scorn. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT IV + + + +SCENE III The forest. + + + [Enter ROSALIND and CELIA] + +ROSALIND How say you now? Is it not past two o'clock? and + here much Orlando! + +CELIA I warrant you, with pure love and troubled brain, he + hath ta'en his bow and arrows and is gone forth to + sleep. Look, who comes here. + + [Enter SILVIUS] + +SILVIUS My errand is to you, fair youth; + My gentle Phebe bid me give you this: + I know not the contents; but, as I guess + By the stern brow and waspish action + Which she did use as she was writing of it, + It bears an angry tenor: pardon me: + I am but as a guiltless messenger. + +ROSALIND Patience herself would startle at this letter + And play the swaggerer; bear this, bear all: + She says I am not fair, that I lack manners; + She calls me proud, and that she could not love me, + Were man as rare as phoenix. 'Od's my will! + Her love is not the hare that I do hunt: + Why writes she so to me? Well, shepherd, well, + This is a letter of your own device. + +SILVIUS No, I protest, I know not the contents: + Phebe did write it. + +ROSALIND Come, come, you are a fool + And turn'd into the extremity of love. + I saw her hand: she has a leathern hand. + A freestone-colour'd hand; I verily did think + That her old gloves were on, but 'twas her hands: + She has a huswife's hand; but that's no matter: + I say she never did invent this letter; + This is a man's invention and his hand. + +SILVIUS Sure, it is hers. + +ROSALIND Why, 'tis a boisterous and a cruel style. + A style for-challengers; why, she defies me, + Like Turk to Christian: women's gentle brain + Could not drop forth such giant-rude invention + Such Ethiope words, blacker in their effect + Than in their countenance. Will you hear the letter? + +SILVIUS So please you, for I never heard it yet; + Yet heard too much of Phebe's cruelty. + +ROSALIND She Phebes me: mark how the tyrant writes. + + [Reads] + + Art thou god to shepherd turn'd, + That a maiden's heart hath burn'd? + Can a woman rail thus? + +SILVIUS Call you this railing? + +ROSALIND [Reads] + + Why, thy godhead laid apart, + Warr'st thou with a woman's heart? + Did you ever hear such railing? + Whiles the eye of man did woo me, + That could do no vengeance to me. + Meaning me a beast. + If the scorn of your bright eyne + Have power to raise such love in mine, + Alack, in me what strange effect + Would they work in mild aspect! + Whiles you chid me, I did love; + How then might your prayers move! + He that brings this love to thee + Little knows this love in me: + And by him seal up thy mind; + Whether that thy youth and kind + Will the faithful offer take + Of me and all that I can make; + Or else by him my love deny, + And then I'll study how to die. + +SILVIUS Call you this chiding? + +CELIA Alas, poor shepherd! + +ROSALIND Do you pity him? no, he deserves no pity. Wilt + thou love such a woman? What, to make thee an + instrument and play false strains upon thee! not to + be endured! Well, go your way to her, for I see + love hath made thee a tame snake, and say this to + her: that if she love me, I charge her to love + thee; if she will not, I will never have her unless + thou entreat for her. If you be a true lover, + hence, and not a word; for here comes more company. + + [Exit SILVIUS] + + [Enter OLIVER] + +OLIVER Good morrow, fair ones: pray you, if you know, + Where in the purlieus of this forest stands + A sheep-cote fenced about with olive trees? + +CELIA West of this place, down in the neighbour bottom: + The rank of osiers by the murmuring stream + Left on your right hand brings you to the place. + But at this hour the house doth keep itself; + There's none within. + +OLIVER If that an eye may profit by a tongue, + Then should I know you by description; + Such garments and such years: 'The boy is fair, + Of female favour, and bestows himself + Like a ripe sister: the woman low + And browner than her brother.' Are not you + The owner of the house I did inquire for? + +CELIA It is no boast, being ask'd, to say we are. + +OLIVER Orlando doth commend him to you both, + And to that youth he calls his Rosalind + He sends this bloody napkin. Are you he? + +ROSALIND I am: what must we understand by this? + +OLIVER Some of my shame; if you will know of me + What man I am, and how, and why, and where + This handkercher was stain'd. + +CELIA I pray you, tell it. + +OLIVER When last the young Orlando parted from you + He left a promise to return again + Within an hour, and pacing through the forest, + Chewing the food of sweet and bitter fancy, + Lo, what befell! he threw his eye aside, + And mark what object did present itself: + Under an oak, whose boughs were moss'd with age + And high top bald with dry antiquity, + A wretched ragged man, o'ergrown with hair, + Lay sleeping on his back: about his neck + A green and gilded snake had wreathed itself, + Who with her head nimble in threats approach'd + The opening of his mouth; but suddenly, + Seeing Orlando, it unlink'd itself, + And with indented glides did slip away + Into a bush: under which bush's shade + A lioness, with udders all drawn dry, + Lay couching, head on ground, with catlike watch, + When that the sleeping man should stir; for 'tis + The royal disposition of that beast + To prey on nothing that doth seem as dead: + This seen, Orlando did approach the man + And found it was his brother, his elder brother. + +CELIA O, I have heard him speak of that same brother; + And he did render him the most unnatural + That lived amongst men. + +OLIVER And well he might so do, + For well I know he was unnatural. + +ROSALIND But, to Orlando: did he leave him there, + Food to the suck'd and hungry lioness? + +OLIVER Twice did he turn his back and purposed so; + But kindness, nobler ever than revenge, + And nature, stronger than his just occasion, + Made him give battle to the lioness, + Who quickly fell before him: in which hurtling + From miserable slumber I awaked. + +CELIA Are you his brother? + +ROSALIND Wast you he rescued? + +CELIA Was't you that did so oft contrive to kill him? + +OLIVER 'Twas I; but 'tis not I I do not shame + To tell you what I was, since my conversion + So sweetly tastes, being the thing I am. + +ROSALIND But, for the bloody napkin? + +OLIVER By and by. + When from the first to last betwixt us two + Tears our recountments had most kindly bathed, + As how I came into that desert place:-- + In brief, he led me to the gentle duke, + Who gave me fresh array and entertainment, + Committing me unto my brother's love; + Who led me instantly unto his cave, + There stripp'd himself, and here upon his arm + The lioness had torn some flesh away, + Which all this while had bled; and now he fainted + And cried, in fainting, upon Rosalind. + Brief, I recover'd him, bound up his wound; + And, after some small space, being strong at heart, + He sent me hither, stranger as I am, + To tell this story, that you might excuse + His broken promise, and to give this napkin + Dyed in his blood unto the shepherd youth + That he in sport doth call his Rosalind. + + [ROSALIND swoons] + +CELIA Why, how now, Ganymede! sweet Ganymede! + +OLIVER Many will swoon when they do look on blood. + +CELIA There is more in it. Cousin Ganymede! + +OLIVER Look, he recovers. + +ROSALIND I would I were at home. + +CELIA We'll lead you thither. + I pray you, will you take him by the arm? + +OLIVER Be of good cheer, youth: you a man! you lack a + man's heart. + +ROSALIND I do so, I confess it. Ah, sirrah, a body would + think this was well counterfeited! I pray you, tell + your brother how well I counterfeited. Heigh-ho! + +OLIVER This was not counterfeit: there is too great + testimony in your complexion that it was a passion + of earnest. + +ROSALIND Counterfeit, I assure you. + +OLIVER Well then, take a good heart and counterfeit to be a man. + +ROSALIND So I do: but, i' faith, I should have been a woman by right. + +CELIA Come, you look paler and paler: pray you, draw + homewards. Good sir, go with us. + +OLIVER That will I, for I must bear answer back + How you excuse my brother, Rosalind. + +ROSALIND I shall devise something: but, I pray you, commend + my counterfeiting to him. Will you go? + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT V + + + +SCENE I The forest. + + + [Enter TOUCHSTONE and AUDREY] + +TOUCHSTONE We shall find a time, Audrey; patience, gentle Audrey. + +AUDREY Faith, the priest was good enough, for all the old + gentleman's saying. + +TOUCHSTONE A most wicked Sir Oliver, Audrey, a most vile + Martext. But, Audrey, there is a youth here in the + forest lays claim to you. + +AUDREY Ay, I know who 'tis; he hath no interest in me in + the world: here comes the man you mean. + +TOUCHSTONE It is meat and drink to me to see a clown: by my + troth, we that have good wits have much to answer + for; we shall be flouting; we cannot hold. + + [Enter WILLIAM] + +WILLIAM Good even, Audrey. + +AUDREY God ye good even, William. + +WILLIAM And good even to you, sir. + +TOUCHSTONE Good even, gentle friend. Cover thy head, cover thy + head; nay, prithee, be covered. How old are you, friend? + +WILLIAM Five and twenty, sir. + +TOUCHSTONE A ripe age. Is thy name William? + +WILLIAM William, sir. + +TOUCHSTONE A fair name. Wast born i' the forest here? + +WILLIAM Ay, sir, I thank God. + +TOUCHSTONE 'Thank God;' a good answer. Art rich? + +WILLIAM Faith, sir, so so. + +TOUCHSTONE 'So so' is good, very good, very excellent good; and + yet it is not; it is but so so. Art thou wise? + +WILLIAM Ay, sir, I have a pretty wit. + +TOUCHSTONE Why, thou sayest well. I do now remember a saying, + 'The fool doth think he is wise, but the wise man + knows himself to be a fool.' The heathen + philosopher, when he had a desire to eat a grape, + would open his lips when he put it into his mouth; + meaning thereby that grapes were made to eat and + lips to open. You do love this maid? + +WILLIAM I do, sir. + +TOUCHSTONE Give me your hand. Art thou learned? + +WILLIAM No, sir. + +TOUCHSTONE Then learn this of me: to have, is to have; for it + is a figure in rhetoric that drink, being poured out + of a cup into a glass, by filling the one doth empty + the other; for all your writers do consent that ipse + is he: now, you are not ipse, for I am he. + +WILLIAM Which he, sir? + +TOUCHSTONE He, sir, that must marry this woman. Therefore, you + clown, abandon,--which is in the vulgar leave,--the + society,--which in the boorish is company,--of this + female,--which in the common is woman; which + together is, abandon the society of this female, or, + clown, thou perishest; or, to thy better + understanding, diest; or, to wit I kill thee, make + thee away, translate thy life into death, thy + liberty into bondage: I will deal in poison with + thee, or in bastinado, or in steel; I will bandy + with thee in faction; I will o'errun thee with + policy; I will kill thee a hundred and fifty ways: + therefore tremble and depart. + +AUDREY Do, good William. + +WILLIAM God rest you merry, sir. + + [Exit] + + [Enter CORIN] + +CORIN Our master and mistress seeks you; come, away, away! + +TOUCHSTONE Trip, Audrey! trip, Audrey! I attend, I attend. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT V + + + +SCENE II The forest. + + + [Enter ORLANDO and OLIVER] + +ORLANDO Is't possible that on so little acquaintance you + should like her? that but seeing you should love + her? and loving woo? and, wooing, she should + grant? and will you persever to enjoy her? + +OLIVER Neither call the giddiness of it in question, the + poverty of her, the small acquaintance, my sudden + wooing, nor her sudden consenting; but say with me, + I love Aliena; say with her that she loves me; + consent with both that we may enjoy each other: it + shall be to your good; for my father's house and all + the revenue that was old Sir Rowland's will I + estate upon you, and here live and die a shepherd. + +ORLANDO You have my consent. Let your wedding be to-morrow: + thither will I invite the duke and all's contented + followers. Go you and prepare Aliena; for look + you, here comes my Rosalind. + + [Enter ROSALIND] + +ROSALIND God save you, brother. + +OLIVER And you, fair sister. + + [Exit] + +ROSALIND O, my dear Orlando, how it grieves me to see thee + wear thy heart in a scarf! + +ORLANDO It is my arm. + +ROSALIND I thought thy heart had been wounded with the claws + of a lion. + +ORLANDO Wounded it is, but with the eyes of a lady. + +ROSALIND Did your brother tell you how I counterfeited to + swoon when he showed me your handkerchief? + +ORLANDO Ay, and greater wonders than that. + +ROSALIND O, I know where you are: nay, 'tis true: there was + never any thing so sudden but the fight of two rams + and Caesar's thrasonical brag of 'I came, saw, and + overcame:' for your brother and my sister no sooner + met but they looked, no sooner looked but they + loved, no sooner loved but they sighed, no sooner + sighed but they asked one another the reason, no + sooner knew the reason but they sought the remedy; + and in these degrees have they made a pair of stairs + to marriage which they will climb incontinent, or + else be incontinent before marriage: they are in + the very wrath of love and they will together; clubs + cannot part them. + +ORLANDO They shall be married to-morrow, and I will bid the + duke to the nuptial. But, O, how bitter a thing it + is to look into happiness through another man's + eyes! By so much the more shall I to-morrow be at + the height of heart-heaviness, by how much I shall + think my brother happy in having what he wishes for. + +ROSALIND Why then, to-morrow I cannot serve your turn for Rosalind? + +ORLANDO I can live no longer by thinking. + +ROSALIND I will weary you then no longer with idle talking. + Know of me then, for now I speak to some purpose, + that I know you are a gentleman of good conceit: I + speak not this that you should bear a good opinion + of my knowledge, insomuch I say I know you are; + neither do I labour for a greater esteem than may in + some little measure draw a belief from you, to do + yourself good and not to grace me. Believe then, if + you please, that I can do strange things: I have, + since I was three year old, conversed with a + magician, most profound in his art and yet not + damnable. If you do love Rosalind so near the heart + as your gesture cries it out, when your brother + marries Aliena, shall you marry her: I know into + what straits of fortune she is driven; and it is + not impossible to me, if it appear not inconvenient + to you, to set her before your eyes tomorrow human + as she is and without any danger. + +ORLANDO Speakest thou in sober meanings? + +ROSALIND By my life, I do; which I tender dearly, though I + say I am a magician. Therefore, put you in your + best array: bid your friends; for if you will be + married to-morrow, you shall, and to Rosalind, if you will. + + [Enter SILVIUS and PHEBE] + + Look, here comes a lover of mine and a lover of hers. + +PHEBE Youth, you have done me much ungentleness, + To show the letter that I writ to you. + +ROSALIND I care not if I have: it is my study + To seem despiteful and ungentle to you: + You are there followed by a faithful shepherd; + Look upon him, love him; he worships you. + +PHEBE Good shepherd, tell this youth what 'tis to love. + +SILVIUS It is to be all made of sighs and tears; + And so am I for Phebe. + +PHEBE And I for Ganymede. + +ORLANDO And I for Rosalind. + +ROSALIND And I for no woman. + +SILVIUS It is to be all made of faith and service; + And so am I for Phebe. + +PHEBE And I for Ganymede. + +ORLANDO And I for Rosalind. + +ROSALIND And I for no woman. + +SILVIUS It is to be all made of fantasy, + All made of passion and all made of wishes, + All adoration, duty, and observance, + All humbleness, all patience and impatience, + All purity, all trial, all observance; + And so am I for Phebe. + +PHEBE And so am I for Ganymede. + +ORLANDO And so am I for Rosalind. + +ROSALIND And so am I for no woman. + +PHEBE If this be so, why blame you me to love you? + +SILVIUS If this be so, why blame you me to love you? + +ORLANDO If this be so, why blame you me to love you? + +ROSALIND Who do you speak to, 'Why blame you me to love you?' + +ORLANDO To her that is not here, nor doth not hear. + +ROSALIND Pray you, no more of this; 'tis like the howling + of Irish wolves against the moon. + + [To SILVIUS] + + I will help you, if I can: + + [To PHEBE] + + I would love you, if I could. To-morrow meet me all together. + + [To PHEBE] + + I will marry you, if ever I marry woman, and I'll be + married to-morrow: + + [To ORLANDO] + + I will satisfy you, if ever I satisfied man, and you + shall be married to-morrow: + + [To SILVIUS] + + I will content you, if what pleases you contents + you, and you shall be married to-morrow. + + [To ORLANDO] + + As you love Rosalind, meet: + + [To SILVIUS] + + as you love Phebe, meet: and as I love no woman, + I'll meet. So fare you well: I have left you commands. + +SILVIUS I'll not fail, if I live. + +PHEBE Nor I. + +ORLANDO Nor I. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT V + + + +SCENE III The forest. + + + [Enter TOUCHSTONE and AUDREY] + +TOUCHSTONE To-morrow is the joyful day, Audrey; to-morrow will + we be married. + +AUDREY I do desire it with all my heart; and I hope it is + no dishonest desire to desire to be a woman of the + world. Here comes two of the banished duke's pages. + + [Enter two Pages] + +First Page Well met, honest gentleman. + +TOUCHSTONE By my troth, well met. Come, sit, sit, and a song. + +Second Page We are for you: sit i' the middle. + +First Page Shall we clap into't roundly, without hawking or + spitting or saying we are hoarse, which are the only + prologues to a bad voice? + +Second Page I'faith, i'faith; and both in a tune, like two + gipsies on a horse. + + SONG. + It was a lover and his lass, + With a hey, and a ho, and a hey nonino, + That o'er the green corn-field did pass + In the spring time, the only pretty ring time, + When birds do sing, hey ding a ding, ding: + Sweet lovers love the spring. + + Between the acres of the rye, + With a hey, and a ho, and a hey nonino + These pretty country folks would lie, + In spring time, &c. + + This carol they began that hour, + With a hey, and a ho, and a hey nonino, + How that a life was but a flower + In spring time, &c. + + And therefore take the present time, + With a hey, and a ho, and a hey nonino; + For love is crowned with the prime + In spring time, &c. + +TOUCHSTONE Truly, young gentlemen, though there was no great + matter in the ditty, yet the note was very + untuneable. + +First Page You are deceived, sir: we kept time, we lost not our time. + +TOUCHSTONE By my troth, yes; I count it but time lost to hear + such a foolish song. God be wi' you; and God mend + your voices! Come, Audrey. + + [Exeunt] + + + + + AS YOU LIKE IT + + +ACT V + + + +SCENE IV The forest. + + + [Enter DUKE SENIOR, AMIENS, JAQUES, ORLANDO, OLIVER, + and CELIA] + +DUKE SENIOR Dost thou believe, Orlando, that the boy + Can do all this that he hath promised? + +ORLANDO I sometimes do believe, and sometimes do not; + As those that fear they hope, and know they fear. + + [Enter ROSALIND, SILVIUS, and PHEBE] + +ROSALIND Patience once more, whiles our compact is urged: + You say, if I bring in your Rosalind, + You will bestow her on Orlando here? + +DUKE SENIOR That would I, had I kingdoms to give with her. + +ROSALIND And you say, you will have her, when I bring her? + +ORLANDO That would I, were I of all kingdoms king. + +ROSALIND You say, you'll marry me, if I be willing? + +PHEBE That will I, should I die the hour after. + +ROSALIND But if you do refuse to marry me, + You'll give yourself to this most faithful shepherd? + +PHEBE So is the bargain. + +ROSALIND You say, that you'll have Phebe, if she will? + +SILVIUS Though to have her and death were both one thing. + +ROSALIND I have promised to make all this matter even. + Keep you your word, O duke, to give your daughter; + You yours, Orlando, to receive his daughter: + Keep your word, Phebe, that you'll marry me, + Or else refusing me, to wed this shepherd: + Keep your word, Silvius, that you'll marry her. + If she refuse me: and from hence I go, + To make these doubts all even. + + [Exeunt ROSALIND and CELIA] + +DUKE SENIOR I do remember in this shepherd boy + Some lively touches of my daughter's favour. + +ORLANDO My lord, the first time that I ever saw him + Methought he was a brother to your daughter: + But, my good lord, this boy is forest-born, + And hath been tutor'd in the rudiments + Of many desperate studies by his uncle, + Whom he reports to be a great magician, + Obscured in the circle of this forest. + + [Enter TOUCHSTONE and AUDREY] + +JAQUES There is, sure, another flood toward, and these + couples are coming to the ark. Here comes a pair of + very strange beasts, which in all tongues are called fools. + +TOUCHSTONE Salutation and greeting to you all! + +JAQUES Good my lord, bid him welcome: this is the + motley-minded gentleman that I have so often met in + the forest: he hath been a courtier, he swears. + +TOUCHSTONE If any man doubt that, let him put me to my + purgation. I have trod a measure; I have flattered + a lady; I have been politic with my friend, smooth + with mine enemy; I have undone three tailors; I have + had four quarrels, and like to have fought one. + +JAQUES And how was that ta'en up? + +TOUCHSTONE Faith, we met, and found the quarrel was upon the + seventh cause. + +JAQUES How seventh cause? Good my lord, like this fellow. + +DUKE SENIOR I like him very well. + +TOUCHSTONE God 'ild you, sir; I desire you of the like. I + press in here, sir, amongst the rest of the country + copulatives, to swear and to forswear: according as + marriage binds and blood breaks: a poor virgin, + sir, an ill-favoured thing, sir, but mine own; a poor + humour of mine, sir, to take that that no man else + will: rich honesty dwells like a miser, sir, in a + poor house; as your pearl in your foul oyster. + +DUKE SENIOR By my faith, he is very swift and sententious. + +TOUCHSTONE According to the fool's bolt, sir, and such dulcet diseases. + +JAQUES But, for the seventh cause; how did you find the + quarrel on the seventh cause? + +TOUCHSTONE Upon a lie seven times removed:--bear your body more + seeming, Audrey:--as thus, sir. I did dislike the + cut of a certain courtier's beard: he sent me word, + if I said his beard was not cut well, he was in the + mind it was: this is called the Retort Courteous. + If I sent him word again 'it was not well cut,' he + would send me word, he cut it to please himself: + this is called the Quip Modest. If again 'it was + not well cut,' he disabled my judgment: this is + called the Reply Churlish. If again 'it was not + well cut,' he would answer, I spake not true: this + is called the Reproof Valiant. If again 'it was not + well cut,' he would say I lied: this is called the + Counter-cheque Quarrelsome: and so to the Lie + Circumstantial and the Lie Direct. + +JAQUES And how oft did you say his beard was not well cut? + +TOUCHSTONE I durst go no further than the Lie Circumstantial, + nor he durst not give me the Lie Direct; and so we + measured swords and parted. + +JAQUES Can you nominate in order now the degrees of the lie? + +TOUCHSTONE O sir, we quarrel in print, by the book; as you have + books for good manners: I will name you the degrees. + The first, the Retort Courteous; the second, the + Quip Modest; the third, the Reply Churlish; the + fourth, the Reproof Valiant; the fifth, the + Countercheque Quarrelsome; the sixth, the Lie with + Circumstance; the seventh, the Lie Direct. All + these you may avoid but the Lie Direct; and you may + avoid that too, with an If. I knew when seven + justices could not take up a quarrel, but when the + parties were met themselves, one of them thought but + of an If, as, 'If you said so, then I said so;' and + they shook hands and swore brothers. Your If is the + only peacemaker; much virtue in If. + +JAQUES Is not this a rare fellow, my lord? he's as good at + any thing and yet a fool. + +DUKE SENIOR He uses his folly like a stalking-horse and under + the presentation of that he shoots his wit. + + [Enter HYMEN, ROSALIND, and CELIA] + + [Still Music] + +HYMEN Then is there mirth in heaven, + When earthly things made even + Atone together. + Good duke, receive thy daughter + Hymen from heaven brought her, + Yea, brought her hither, + That thou mightst join her hand with his + Whose heart within his bosom is. + +ROSALIND [To DUKE SENIOR] To you I give myself, for I am yours. + + [To ORLANDO] + + To you I give myself, for I am yours. + +DUKE SENIOR If there be truth in sight, you are my daughter. + +ORLANDO If there be truth in sight, you are my Rosalind. + +PHEBE If sight and shape be true, + Why then, my love adieu! + +ROSALIND I'll have no father, if you be not he: + I'll have no husband, if you be not he: + Nor ne'er wed woman, if you be not she. + +HYMEN Peace, ho! I bar confusion: + 'Tis I must make conclusion + Of these most strange events: + Here's eight that must take hands + To join in Hymen's bands, + If truth holds true contents. + You and you no cross shall part: + You and you are heart in heart + You to his love must accord, + Or have a woman to your lord: + You and you are sure together, + As the winter to foul weather. + Whiles a wedlock-hymn we sing, + Feed yourselves with questioning; + That reason wonder may diminish, + How thus we met, and these things finish. + + SONG. + Wedding is great Juno's crown: + O blessed bond of board and bed! + 'Tis Hymen peoples every town; + High wedlock then be honoured: + Honour, high honour and renown, + To Hymen, god of every town! + +DUKE SENIOR O my dear niece, welcome thou art to me! + Even daughter, welcome, in no less degree. + +PHEBE I will not eat my word, now thou art mine; + Thy faith my fancy to thee doth combine. + + [Enter JAQUES DE BOYS] + +JAQUES DE BOYS Let me have audience for a word or two: + I am the second son of old Sir Rowland, + That bring these tidings to this fair assembly. + Duke Frederick, hearing how that every day + Men of great worth resorted to this forest, + Address'd a mighty power; which were on foot, + In his own conduct, purposely to take + His brother here and put him to the sword: + And to the skirts of this wild wood he came; + Where meeting with an old religious man, + After some question with him, was converted + Both from his enterprise and from the world, + His crown bequeathing to his banish'd brother, + And all their lands restored to them again + That were with him exiled. This to be true, + I do engage my life. + +DUKE SENIOR Welcome, young man; + Thou offer'st fairly to thy brothers' wedding: + To one his lands withheld, and to the other + A land itself at large, a potent dukedom. + First, in this forest, let us do those ends + That here were well begun and well begot: + And after, every of this happy number + That have endured shrewd days and nights with us + Shall share the good of our returned fortune, + According to the measure of their states. + Meantime, forget this new-fall'n dignity + And fall into our rustic revelry. + Play, music! And you, brides and bridegrooms all, + With measure heap'd in joy, to the measures fall. + +JAQUES Sir, by your patience. If I heard you rightly, + The duke hath put on a religious life + And thrown into neglect the pompous court? + +JAQUES DE BOYS He hath. + +JAQUES To him will I : out of these convertites + There is much matter to be heard and learn'd. + + [To DUKE SENIOR] + + You to your former honour I bequeath; + Your patience and your virtue well deserves it: + + [To ORLANDO] + + You to a love that your true faith doth merit: + + [To OLIVER] + + You to your land and love and great allies: + + [To SILVIUS] + + You to a long and well-deserved bed: + + [To TOUCHSTONE] + + And you to wrangling; for thy loving voyage + Is but for two months victuall'd. So, to your pleasures: + I am for other than for dancing measures. + +DUKE SENIOR Stay, Jaques, stay. + +JAQUES To see no pastime I what you would have + I'll stay to know at your abandon'd cave. + + [Exit] + +DUKE SENIOR Proceed, proceed: we will begin these rites, + As we do trust they'll end, in true delights. + + [A dance] + + + + + AS YOU LIKE IT + + EPILOGUE + + +ROSALIND It is not the fashion to see the lady the epilogue; + but it is no more unhandsome than to see the lord + the prologue. If it be true that good wine needs + no bush, 'tis true that a good play needs no + epilogue; yet to good wine they do use good bushes, + and good plays prove the better by the help of good + epilogues. What a case am I in then, that am + neither a good epilogue nor cannot insinuate with + you in the behalf of a good play! I am not + furnished like a beggar, therefore to beg will not + become me: my way is to conjure you; and I'll begin + with the women. I charge you, O women, for the love + you bear to men, to like as much of this play as + please you: and I charge you, O men, for the love + you bear to women--as I perceive by your simpering, + none of you hates them--that between you and the + women the play may please. If I were a woman I + would kiss as many of you as had beards that pleased + me, complexions that liked me and breaths that I + defied not: and, I am sure, as many as have good + beards or good faces or sweet breaths will, for my + kind offer, when I make curtsy, bid me farewell. + + [Exeunt] diff --git a/target/snappy-1.1.10/testdata/baddata1.snappy b/target/snappy-1.1.10/testdata/baddata1.snappy new file mode 100644 index 00000000..99d970fb Binary files /dev/null and b/target/snappy-1.1.10/testdata/baddata1.snappy differ diff --git a/target/snappy-1.1.10/testdata/baddata2.snappy b/target/snappy-1.1.10/testdata/baddata2.snappy new file mode 100644 index 00000000..8f5cb13e Binary files /dev/null and b/target/snappy-1.1.10/testdata/baddata2.snappy differ diff --git a/target/snappy-1.1.10/testdata/baddata3.snappy b/target/snappy-1.1.10/testdata/baddata3.snappy new file mode 100644 index 00000000..774aead4 Binary files /dev/null and b/target/snappy-1.1.10/testdata/baddata3.snappy differ diff --git a/target/snappy-1.1.10/testdata/fireworks.jpeg b/target/snappy-1.1.10/testdata/fireworks.jpeg new file mode 100644 index 00000000..078cf175 Binary files /dev/null and b/target/snappy-1.1.10/testdata/fireworks.jpeg differ diff --git a/target/snappy-1.1.10/testdata/geo.protodata b/target/snappy-1.1.10/testdata/geo.protodata new file mode 100644 index 00000000..c4e3e0d8 Binary files /dev/null and b/target/snappy-1.1.10/testdata/geo.protodata differ diff --git a/target/snappy-1.1.10/testdata/html b/target/snappy-1.1.10/testdata/html new file mode 100644 index 00000000..ef768ccd --- /dev/null +++ b/target/snappy-1.1.10/testdata/html @@ -0,0 +1 @@ + content: @ 1099872000000000: 'HTTP/1.1 200 OK\r\nX-Google-Crawl-Date: Mon, 08 Nov 2004 17:22:09 GMT\r\nContent-Type: text/html\r\nConnection: close\r\nX-Powered-By: PHP/4.3.8\r\nServer: Apache/1.3.31 (Unix) mod_gzip/1.3.19.1a PHP/4.3.8\r\nDate: Mon, 08 Nov 2004 17:19:07 GMT\r\n\r\n \r\n\r\n\r\n\r\n\r\n\nMicro Achat : Ordinateurs, PDA - Toute l\'informatique avec 01Informatique, L\'Ordinateur Individuel, Micro Hebdo, D\351cision Informatique et 01R\351seaux\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n

T\351l\351charger des bandes-annonces, des teasers, des extraits et des making of de vos films cin\351ma pr\351f\351r\351sPortail leader de l\'information sur les nouvelles technologies - Actualites et enquetes sur l\'informatique et les NTIC (nouvelles technologies de l\'information et la communication). Exemples de sujets : e-business, telecoms, CRM, VPN, 3G, DSL, fournisseurs d\'acces, Linux, piratage, securite informatique, progiciel...s\351curit\351 informatiquestrat\351gies NTICpiratage informatiquetechnologie DSLsolutions VPNstrat\351giesoutils CRMlogiciel Linuxfournisseur d\'acc\350sprogicielstockage de donn\351es\351tudes solution informatiqueNTICactualit\351 technologie\351tude mat\351riel informatiqueactualit\351s des nouvelles technologiesenqu\352tes et trucs et astuces sur le mat\351riel informatique, les logiciels, les fournisseurs d\'acc\350s...emploi et formations informatiques : offres d\'emploi informatique...fiches, tests mat\351riels et logiciels en informatique01 InformatiqueCaract\350re, le site des professionnels de l\'imprim\351 - communication imprim\351e, \351dition, emballage, encres, finition, flexographie, h\351liogravure, impression, imprimerie, imprimeur, industries graphiques, num\351rique, offset, papier, pr\351presse, presse, print, reprographie, s\351rigraphie, typographieDecision MicroD\351cision Informatique01 R\351seauxlogicield\351veloppeur programmationanti-virusgravure cd dvdcodec encodeur d\351codeursoftwarelogicielsgratuitp2p peer to peer partage de fichierss\351curit\351compression et d\351compressiondivxutilitairejeuxloisirsmaclinuxsharewarevid\351o dvdmultim\351diatraductionmp3tchatchatmusiquemobilet\351l\351phone portablet\351l\351phone mobile

\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n

Derni\350re mise \340 jour de cette page : lundi 8 novembre 2004 | 16:45

\r\n \r\n\r\n\r\n\r\n\t\r\n\r\n\t\t\r\n\r\n\t\r\n\r\n\t\r\n\t\t\r\n\r\n\r\n\t\t\r\n\r\n\t\t\r\n\t\r\n\t\r\n\t\t\t

\r\n\r\n\r\n\r\n

\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t

$\"\"$
Imagerie
$\"\"$	\n\t\t\t\t\t\t\t\tLG L1720B\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 332.89 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t
Ordinateurs
$\"\"$	\n\t\t\t\t\t\t\t\tAcer Veriton 7600G\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 705 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t
Ordinateurs
$\"\"$	\n\t\t\t\t\t\t\t\tShuttle SN95G5\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 375 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t
Composants
$\"\"$	\n\t\t\t\t\t\t\t\tAsus A7N8X-E Deluxe\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 91.99 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t
Composants
$\"\"$	\n\t\t\t\t\t\t\t\tThermalright SP-94\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 49 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t

\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t

$\"\"$
1 $\">\"$	PC Look
2 $\">\"$	Atelier Informatique
3 $\">\"$	Zanax Multim\351dia
4 $\">\"$	MISTEROOPS
5 $\">\"$	168 Golden Avenue
6 $\">\"$	microchoix
7 $\">\"$	e-Soph
8 $\">\"$	PC Price Club
9 $\">\"$	PC 77
10 $\">\"$	Web In Informatique

\n\t\t\t\t

\n\t\t\t\t
\r\n \r\n\r\n\r\n\r\n\r\n\r\n

\r\n\t\t\r\n\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\t

\n\n\n\n\n\n\n\n\n\n\n\n\n


	\n\n \n \n \n \n \n \n \n \n \n \n

\r\n\r\n\r\n\t\t\r\n\t\t\t\r\n\t\t\r\n

\r\n\t\t\t

\r\n

\r\n\r\n \r\n \r\n\t\r\n\t\t\r\n\t\t\r\n\t\t\r\n\t\r\n

\r\n\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\t\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t


	CD et DVD bient\364t insensibles aux rayures
	OpenOffice gagne son service
	La messagerie en cinq minutes selon Ipswitch
> toutes les news

\r\n\t\t

\r\n

\r\n\r\n\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\t\r\n\t\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\t\r\n\t\t\r\n\t\r\n

\r\n\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t

Recevez chaque jour l\'actualit\351 des produits et des promos
\r\n\t\t\t\t\r\n\t\t\t\t\t\r\n\t\t\t\t\t\t\r\n\t\t\t\t\t\r\n\t\t\t\t

\r\n\t\t\t

\r\n\t\t\t\t

\r\n\t\t

\r\n

\r\n\r\n\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\t\r\n\t\r\n

\r\n\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t \r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\r\n\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t


	Entreprise \r\n\t\t\t\tQuand le billet papier s\'envole vers la d\351mat\351rialisation

	Trucs et astuces \r\n\t\t\t\tD\351pannez Windows XP

	Conso \r\n\t\t\t\tVos photos sur papier imprimante ou labo ?

	Produits & Tests \r\n\t\t\t\t5 programmes d\222encodage vid\351o gratuits

\r\n\t\t

\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n

\r\n
\r\n

\r\n\t\t\r\n\t\t

\r\n\t\t\r\n\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\t\r\n\t\t

\r\n

\r\n\r\n\r\n\t\r\n\t\t\r\n\t\r\n

\r\n\t

\r\n\t\t\r\n\t\t\r\n\r\n\r\n\r\n \r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\t\r\n\t\t

\r\n

\r\n\r\n\r\n\t\r\n\r\n\r\n\r\n\t\t\t\r\n\r\n\r\n\r\n\t\t\t\t\r\n

\r\n

\r\nPortable
\r\nUn nouvel ultra portable r\351alis\351 par Nec
\r\n\r\n\t\t\t\t\t \t \t\t\t\t\t\t\t\t\t\t\t\t\t

\r\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t \t \t\r\n

Le Versa S940 a un format r\351duit, mais ses performances sont \340 la hauteur.
\r\n\340 partir de 1663 \200\r\n

\r\n

\r\nPortable
\r\nAsus pr\351sente trois petits nouveaux dans la gamme A3N
\r\n\r\n\t\t\t\t\t \t \t\t\t\t\t\t\t\t\t\t\t\t\t

\r\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t \t \t\r\n

Ces trois portables Centrino int\350grent, entre autres, une webcam et un contr\364leur Wi-Fi.
\r\n\340 partir de 1346 \200\r\n

\r\n

\r\n\t\r\n\t\r\n\t\r\n \t\r\n\t\r\n\t\r\n \t\r\n\t\r\n\t

\r\n\t\r\n

\r\n

\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n

BON PLAN

\r\n

\r\nLes derni\350res technologies INTEL dans un nouveau design pour ce shuttle haut de gamme, pour un prix abordable.
\r\n

\r\n\r\n\340 partir de
\r\n

415 \200

\r\n\r\n

\r\n

publicit\351

\r\n

\r\n\t\r\n

\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n

\r\n\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t

\r\n\t\t\t\t\t\t\t\t\t\t\t\t

\r\n\t\t\t\t

Desktops
\r\n\t\t\t\t

Portables
\r\n\t\t\t\t

Mini-PC
\r\n\t\t\t\t

Pda / Tablets-PC
\r\n\t\t\t\t

Apple
\r\n\t\t\t\t

GPS
\r\n\t\t\t\t

\r\n\t\t\t\t\t\t\t\t\r\n\t\t\t\t\t\r\n\t\t\t\t\t\t\r\n\t\t\t\t\t\r\n\t\t\t\t

\r\n\t\t\t\t\t\t

\r\n\t\t\t\t

\r\n\t\t

\r\n

\r\n\t\t\r\n\t\t\r\n\r\n\r\n \r\n\r\n\r\n \r\n\r\n\r\n\r\n\t\t\r\n\t\t

\r\n

\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n

\r\n\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\t\t\r\n\t\r\n\t\r\n\t\t\t\t\r\n\t\r\n\t\r\n\t\t\t\t\r\n\t\r\n\t\r\n\t\t\t\t\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t

\r\n\tPortable Toshiba consacre deux gammes de machines au multim\351dia
\r\n\tEquipement haut de gamme et Windows Media Center sont au menu de ces portables \340 vocation multim\351dia.

\r\n\tOrdinateur Arriv\351e d\'un Power Mac G5 d\'entr\351e de gamme
\r\n\tLa firme \340 la pomme propose une station de travail \351volutive et relativement abordable.

\r\n\tPC Alienware propose deux machines au look \351trange
\r\n\tAurora et Area 51 sont deux gammes d\'ordinateurs enti\350rement configurables.

\r\n\tPortable Trois nouveaux iBook G4 chez Apple
\r\n\tChez Apple, les portables gagnent en vitesse et communiquent sans fil en standard.

\r\n\t\t\t\t> toutes les news\r\n\t\t\t

\r\n

\r\n\r\n\r\n\r\n\r\n

\r\n\r\n\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n

\r\n\t\t\t\t\t \t \t\t\t\t\t\t\t\t\t\t\t\t\t

\r\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t \t \t

Asus A3N15-C Pro
\r\n

Voici un portable autonome et puissant gr\342ce \340 la technologie Intel Centrino.

\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t


	\r\n\t1170 \200

\r\n

\r\n\t\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n

\r\n\t\t\t\t\t \t \t\t\t\t\t\t\t\t\t\t\t\t\t

\r\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t \t \t

Soltek EQ3702A Miroir
\r\n

Ce mini PC est une solution int\351ressante pour les utilisateurs poss\351dant d\351j\340 un \351cran.

\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t


	\r\n\t559 \200

\r\n

\r\n\t\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n

\r\n\t\t\t\t\t \t \t\t\t\t\t\t\t\t\t\t\t\t\t

\r\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t \t \t

IBM ThinkPad R51
\r\n

Voici un portable complet et pourtant relativement l\351ger.

\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t\r\n\t


	\r\n\t1299 \200

\r\n

\r\n\t\t\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n

\r\n\t\t\t\t> toutes les promos\r\n\t\t

\r\n

\r\n\t\t\r\n\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\t\r\n\t\t

\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n

\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\r\n

\r\n

\r\n\t\t\r\n\t\t\r\n\t\t\t\r\n\t\t\r\n\t\t\r\n\t\t

\r\n\t\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\t\t

\r\n\t\t\t\r\n\t\t\t\t\r\n\r\n\r\n\r\n \r\n\r\n\r\n \r\n\r\n\r\n\r\n\r\n\t\t\r\n\t\t

\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\r\n


\r\n\t\t \r\n\t\r\n\r\n\r\n\r\n\t\t\t\t\t \t \t\t\t\t\t\t\t\t\t\t\t\t\t\r\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t \t \t\r\n\r\n \r\nLes graveurs de DVD \r\nQuel graveur choisir ? Quel type de format ? Quelle vitesse ? Double couche ou simple couche ? Voici tout ce qu\'il faut savoir pour faire le bon choix. \r\n\t\t\t\t\t\t \r\n\t\t

\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\r\n

\r\n\t\t

\r\n

\t\t\t\t\t\t\tKodak EasyShare Printer Dock Plus :\r\n\t\t\t\t\t\tvos photos 10 x 15 cm en 60 secondes

\r\n\t\t\t\t

\r\n

\t\t\t\t\t\t\t3DMark05 :\r\n\t\t\t\t\t\tun labo 3D \340 domicile

\r\n\t\t\t\t

\r\n

\t\t\t\t\t\t\tDVDInfoPro 2.6\r\n\t\t\t\t\t\treconna\356t CD et DVD

\r\n\t\t\t\t

\r\n

\t\t\t\t\t\t\tScaleo PA D3008, de Fujitsu-Siemens :\r\n\t\t\t\t\t\tune impression d\'inachev\351

\r\n\t\t\t\t

\r\n

Carbon 5 Go : 2 500 chansons dans le creux de la main

\r\n\t\t\t

\r\n

\r\n\r\n\r\n\t\r\n\t\t\r\n\r\n\r\n \t\r\n

\r\n\t\t

\r\n\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t

\r\n\t\t\t\t

Choisir une r\351gion
\r\n\r\n

Un d\351partement
\r\n\r\n

\r\n

Un arrondissement
\r\n\r\n

\r\n

\r\n\t\t\t\t

Recherche directe
\r\n\t\t\t\t

rechercher une ville

et/ou une boutique
\r\n\t\t\t\t

\r\n\t\t\t\t

\r\n\t\t\t\t\r\n\t\t\t\t\t\r\n\t\t\t\t\t\t\r\n\t\t\t\t\t\r\n\t\t\t\t

Recherche avanc\351e

\r\n\t\t\t\t

\r\n\t\t

\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\r\n

\r\n\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t\t\r\n\t\t\t\t\r\n\t\t\t\r\n\t\t


Bureautique \r\n\t\t\t\tTraducteur, organiseur... \r\n\t\t\t\t

Multim\351dia \r\n\t\t\t\tPhoto, audio, vid\351o... \r\n\t\t\t\t

Utilitaires \r\n\t\t\t\tAntivirus, pilotes, gravure... \r\n\t\t\t\t

Personnaliser son PC \r\n\t\t\t\tEcrans de veille, th\350mes... \r\n\t\t\t\t

D\351veloppement \r\n\t\t\t\tCr\351ation de logiciels, BDD... \r\n\t\t\t\t

Jeux \r\n\t\t\t\tAction, simulation... \r\n\t\t\t\t

Internet \r\n\t\t\t\tUtilitaires, email, FTP... \r\n\t\t\t\t

Loisirs \r\n\t\t\t\tHumour, culture... \r\n\t\t\t\t

\r\n\t\t

\r\n

\r\n\t\t\r\n\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\t\r\n\t\t

\r\n\t\t

\r\n\t\t\r\n\t\t\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\t\t\t\r\n\t\t\t

\r\n\t\t\t

\r\n\r\n\t\t

\r\n\t\t\t\r\n\r\n\r\n\r\n\r\n\r\n

\r\n

\r\n\r\n\r\n\r\n\r\n\nMicro Achat : Ordinateurs, PDA - Toute l\'informatique avec 01Informatique, L\'Ordinateur Individuel, Micro Hebdo, D\351cision Informatique et 01R\351seaux\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n

\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\r\n\t\t\r\n\t\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n

\r\n\r\n\t\r\n\t\t\r\n\t\t\r\n\t\r\n

Derni\350re mise \340 jour de cette page : lundi 8 novembre 2004 | 16:45

\r\n \r\n\r\n\r\n\r\n\t\r\n\r\n\t\t\r\n\r\n\t\r\n\r\n\t\r\n\t\t\r\n\r\n\r\n\t\t\r\n\r\n\t\t\r\n\t\r\n\t\r\n\t\t\t

\r\n\r\n\r\n\r\n

$\"\"$
Imagerie
$\"\"$	\n\t\t\t\t\t\t\t\tLG L1720B\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 332.89 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t
Ordinateurs
$\"\"$	\n\t\t\t\t\t\t\t\tAcer Veriton 7600G\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 705 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t
Ordinateurs
$\"\"$	\n\t\t\t\t\t\t\t\tShuttle SN95G5\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 375 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t
Composants
$\"\"$	\n\t\t\t\t\t\t\t\tAsus A7N8X-E Deluxe\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 91.99 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t
Composants
$\"\"$	\n\t\t\t\t\t\t\t\tThermalright SP-94\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\340 partir de\n\t\t\t\t\t\t\t\t\t 49 € \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t

$\"\"$
1 $\">\"$	PC Look
2 $\">\"$	Atelier Informatique
3 $\">\"$	Zanax Multim\351dia
4 $\">\"$	MISTEROOPS
5 $\">\"$	168 Golden Avenue
6 $\">\"$	microchoix
7 $\">\"$	e-Soph
8 $\">\"$	PC Price Club
9 $\">\"$	PC 77
10 $\">\"$	Web In Informatique

\n\t\t\t\t

\n\t\t\t\t
\r\n \r\n\r\n\r\n\r\n\r\n\r\n

\n\n\n\n\n\n\n\n\n\n\n\n\n


	\n\n \n \n \n \n \n \n \n \n \n \n