monasca-common/monasca_common/kafka_lib/codec.py

import gzip
from io import BytesIO
import struct

from six.moves import xrange

_XERIAL_V1_HEADER = (-126, b'S', b'N', b'A', b'P', b'P', b'Y', 0, 1, 1)
_XERIAL_V1_FORMAT = 'bccccccBii'

try:
    import snappy
    _HAS_SNAPPY = True
except ImportError:
    _HAS_SNAPPY = False


def has_gzip():
    return True


def has_snappy():
    return _HAS_SNAPPY


def gzip_encode(payload, compresslevel=None):
    if not compresslevel:
        compresslevel = 9

    with BytesIO() as buf:

        # Gzip context manager introduced in python 2.6
        # so old-fashioned way until we decide to not support 2.6
        gzipper = gzip.GzipFile(fileobj=buf, mode="w", compresslevel=compresslevel)
        try:
            gzipper.write(payload)
        finally:
            gzipper.close()

        result = buf.getvalue()

    return result


def gzip_decode(payload):
    with BytesIO(payload) as buf:

        # Gzip context manager introduced in python 2.6
        # so old-fashioned way until we decide to not support 2.6
        gzipper = gzip.GzipFile(fileobj=buf, mode='r')
        try:
            result = gzipper.read()
        finally:
            gzipper.close()

    return result


def snappy_encode(payload, xerial_compatible=False, xerial_blocksize=32 * 1024):
    """Encodes the given data with snappy if xerial_compatible is set then the
       stream is encoded in a fashion compatible with the xerial snappy library

       The block size (xerial_blocksize) controls how frequent the blocking
       occurs 32k is the default in the xerial library.

       The format winds up being
        +-------------+------------+--------------+------------+--------------+
        |   Header    | Block1 len | Block1 data  | Blockn len | Blockn data  |
        |-------------+------------+--------------+------------+--------------|
        |  16 bytes   |  BE int32  | snappy bytes |  BE int32  | snappy bytes |
        +-------------+------------+--------------+------------+--------------+

        It is important to not that the blocksize is the amount of uncompressed
        data presented to snappy at each block, whereas the blocklen is the
        number of bytes that will be present in the stream, that is the
        length will always be <= blocksize.
    """

    if not has_snappy():
        raise NotImplementedError("Snappy codec is not available")

    if xerial_compatible:
        def _chunker():
            for i in xrange(0, len(payload), xerial_blocksize):
                yield payload[i:i+xerial_blocksize]

        out = BytesIO()

        header = b''.join([struct.pack('!' + fmt, dat) for fmt, dat
                           in zip(_XERIAL_V1_FORMAT, _XERIAL_V1_HEADER)])

        out.write(header)
        for chunk in _chunker():
            block = snappy.compress(chunk)
            block_size = len(block)
            out.write(struct.pack('!i', block_size))
            out.write(block)

        out.seek(0)
        return out.read()

    else:
        return snappy.compress(payload)


def _detect_xerial_stream(payload):
    """Detects if the data given might have been encoded with the blocking mode
        of the xerial snappy library.

        This mode writes a magic header of the format:
            +--------+--------------+------------+---------+--------+
            | Marker | Magic String | Null / Pad | Version | Compat |
            |--------+--------------+------------+---------+--------|
            |  byte  |   c-string   |    byte    |  int32  | int32  |
            |--------+--------------+------------+---------+--------|
            |  -126  |   'SNAPPY'   |     \0     |         |        |
            +--------+--------------+------------+---------+--------+

        The pad appears to be to ensure that SNAPPY is a valid cstring
        The version is the version of this format as written by xerial,
        in the wild this is currently 1 as such we only support v1.

        Compat is there to claim the miniumum supported version that
        can read a xerial block stream, presently in the wild this is
        1.
    """

    if len(payload) > 16:
        header = struct.unpack('!' + _XERIAL_V1_FORMAT, bytes(payload)[:16])
        return header == _XERIAL_V1_HEADER
    return False


def snappy_decode(payload):
    if not has_snappy():
        raise NotImplementedError("Snappy codec is not available")

    if _detect_xerial_stream(payload):
        # TODO ? Should become a fileobj ?
        out = BytesIO()
        byt = payload[16:]
        length = len(byt)
        cursor = 0

        while cursor < length:
            block_size = struct.unpack_from('!i', byt[cursor:])[0]
            # Skip the block size
            cursor += 4
            end = cursor + block_size
            out.write(snappy.decompress(byt[cursor:end]))
            cursor = end

        out.seek(0)
        return out.read()
    else:
        return snappy.decompress(payload)