path: root/library/cpp/packers/packers.h

#pragma once
 
#include <util/generic/string.h>
#include <util/generic/strbuf.h>
#include <util/generic/set.h>
#include <util/generic/list.h>
#include <util/generic/vector.h>
#include <util/generic/bitops.h>

#include <array>
// Data serialization strategy class. 
// Default realization can pack only limited range of types, but you can pack any data other using your own strategy class. 
 
template <class T> 
class TNullPacker { // Very effective package class - pack any data into zero bytes :)
public: 
    void UnpackLeaf(const char*, T& t) const {
        t = T();
    } 
 
    void PackLeaf(char*, const T&, size_t) const {
    }
 
    size_t MeasureLeaf(const T&) const {
        return 0; 
    } 
 
    size_t SkipLeaf(const char*) const {
        return 0; 
    } 
}; 
 
template <typename T>
class TAsIsPacker { // this packer is not really a packer...
public:
    void UnpackLeaf(const char* p, T& t) const {
        memcpy(&t, p, sizeof(T));
    }
    void PackLeaf(char* buffer, const T& data, size_t computedSize) const {
        Y_ASSERT(computedSize == sizeof(data));
        memcpy(buffer, &data, sizeof(T));
    }
    size_t MeasureLeaf(const T& data) const {
        Y_UNUSED(data);
        return sizeof(T);
    }
    size_t SkipLeaf(const char*) const {
        return sizeof(T);
    }
};

// Implementation 
 
namespace NPackers {
    template <class T> 
    inline ui64 ConvertIntegral(const T& data); 
 
    template <> 
    inline ui64 ConvertIntegral(const i64& data) { 
        if (data < 0) { 
            return (static_cast<ui64>(-1 * data) << 1) | 1; 
        } else { 
            return static_cast<ui64>(data) << 1; 
        } 
    } 
 
    namespace NImpl {
        template <class T, bool isSigned>
        struct TConvertImpl {
            static inline ui64 Convert(const T& data);
        };

        template <class T>
        struct TConvertImpl<T, true> {
            static inline ui64 Convert(const T& data) {
                return ConvertIntegral<i64>(static_cast<i64>(data));
            }
        };

        template <class T>
        struct TConvertImpl<T, false> {
            static inline ui64 Convert(const T& data) {
                return data;
            }
        };
    }

    template <class T> 
    inline ui64 ConvertIntegral(const T& data) { 
        static_assert(std::is_integral<T>::value, "T must be integral type");
        return NImpl::TConvertImpl<T, std::is_signed<T>::value>::Convert(data);
    } 
 
    //---------------------------------
    // TIntegralPacker --- for integral types.

    template <class T> 
    class TIntegralPacker { // can pack only integral types <= ui64
    public: 
        void UnpackLeaf(const char* p, T& t) const;
        void PackLeaf(char* buffer, const T& data, size_t size) const;
        size_t MeasureLeaf(const T& data) const;
        size_t SkipLeaf(const char* p) const;
    }; 
 
    template <> 
    inline size_t TIntegralPacker<ui64>::MeasureLeaf(const ui64& val) const {
        constexpr size_t MAX_SIZE = sizeof(ui64) + sizeof(ui64) / 8;
 
        ui64 value = val; 
        size_t len = 1; 
 
        value >>= 7; 
        for (; value && len < MAX_SIZE; value >>= 7)
            ++len; 
 
        return len; 
    } 
 
    template <> 
    inline void TIntegralPacker<ui64>::PackLeaf(char* buffer, const ui64& val, size_t len) const {
        ui64 value = val; 
        int lenmask = 0; 
 
        for (size_t i = len - 1; i; --i) {
            buffer[i] = (char)(value & 0xFF); 
            value >>= 8; 
            lenmask = ((lenmask >> 1) | (1 << 7)); 
        } 
 
        buffer[0] = (char)(lenmask | value); 
    } 
 
    extern const ui8 SkipTable[];

    template <>
    inline void TIntegralPacker<ui64>::UnpackLeaf(const char* p, ui64& result) const {
        unsigned char ch = *(p++);
        size_t taillen = SkipTable[ch] - 1;

        result = (ch & (0x7F >> taillen));

        while (taillen--)
            result = ((result << 8) | (*(p++) & 0xFF));
    }

    template <> 
    inline size_t TIntegralPacker<ui64>::SkipLeaf(const char* p) const {
        return SkipTable[(ui8)*p];
    } 
 
    namespace NImpl {
        template <class T, bool isSigned>
        struct TUnpackLeafImpl {
            inline void UnpackLeaf(const char* p, T& t) const;
        };
        template <class T>
        struct TUnpackLeafImpl<T, true> {
            inline void UnpackLeaf(const char* p, T& t) const {
                ui64 val;
                TIntegralPacker<ui64>().UnpackLeaf(p, val);
                if (val & 1) {
                    t = -1 * static_cast<i64>(val >> 1);
                } else {
                    t = static_cast<T>(val >> 1);
                }
            }
        };
        template <class T>
        struct TUnpackLeafImpl<T, false> {
            inline void UnpackLeaf(const char* p, T& t) const {
                ui64 tmp;
                TIntegralPacker<ui64>().UnpackLeaf(p, tmp);
                t = static_cast<T>(tmp);
            }
        };
    }

    template <class T> 
    inline void TIntegralPacker<T>::UnpackLeaf(const char* p, T& t) const {
        NImpl::TUnpackLeafImpl<T, std::is_signed<T>::value>().UnpackLeaf(p, t);
    } 
 
    template <class T> 
    inline void TIntegralPacker<T>::PackLeaf(char* buffer, const T& data, size_t size) const {
        TIntegralPacker<ui64>().PackLeaf(buffer, ConvertIntegral<T>(data), size);
    } 
 
    template <class T> 
    inline size_t TIntegralPacker<T>::MeasureLeaf(const T& data) const {
        return TIntegralPacker<ui64>().MeasureLeaf(ConvertIntegral<T>(data));
    } 
 
    template <class T> 
    inline size_t TIntegralPacker<T>::SkipLeaf(const char* p) const {
        return TIntegralPacker<ui64>().SkipLeaf(p);
    } 
 
    //-------------------------------------------
    // TFPPacker --- for float/double
    namespace NImpl {
        template <class TFloat, class TUInt>
        class TFPPackerBase {
        protected:
            typedef TIntegralPacker<TUInt> TPacker;

            union THelper {
                TFloat F;
                TUInt U;
            };

            TFloat FromUInt(TUInt u) const {
                THelper h;
                h.U = ReverseBytes(u);
                return h.F;
            }

            TUInt ToUInt(TFloat f) const {
                THelper h;
                h.F = f;
                return ReverseBytes(h.U);
            }

        public:
            void UnpackLeaf(const char* c, TFloat& t) const {
                TUInt u = 0;
                TPacker().UnpackLeaf(c, u);
                t = FromUInt(u);
            }

            void PackLeaf(char* c, const TFloat& t, size_t sz) const {
                TPacker().PackLeaf(c, ToUInt(t), sz);
            }

            size_t MeasureLeaf(const TFloat& t) const {
                return TPacker().MeasureLeaf(ToUInt(t));
            }

            size_t SkipLeaf(const char* c) const {
                return TPacker().SkipLeaf(c);
            }
        };
    }

    class TFloatPacker: public NImpl::TFPPackerBase<float, ui32> {
    };

    class TDoublePacker: public NImpl::TFPPackerBase<double, ui64> {
    };

    //-------------------------------------------
    // TStringPacker --- for TString/TUtf16String and TStringBuf.

    template <class TStringType>
    class TStringPacker {
    public: 
        void UnpackLeaf(const char* p, TStringType& t) const;
        void PackLeaf(char* buffer, const TStringType& data, size_t size) const;
        size_t MeasureLeaf(const TStringType& data) const;
        size_t SkipLeaf(const char* p) const;
    }; 
 
    template <class TStringType>
    inline void TStringPacker<TStringType>::UnpackLeaf(const char* buf, TStringType& t) const {
        size_t len;
        TIntegralPacker<size_t>().UnpackLeaf(buf, len);
        size_t start = TIntegralPacker<size_t>().SkipLeaf(buf);
        t = TStringType((const typename TStringType::char_type*)(buf + start), len);
    } 
 
    template <class TStringType>
    inline void TStringPacker<TStringType>::PackLeaf(char* buf, const TStringType& str, size_t size) const {
        size_t len = str.size(); 
        size_t lenChar = len * sizeof(typename TStringType::char_type);
        size_t start = size - lenChar; 
        TIntegralPacker<size_t>().PackLeaf(buf, len, TIntegralPacker<size_t>().MeasureLeaf(len));
        memcpy(buf + start, str.data(), lenChar);
    } 
 
    template <class TStringType>
    inline size_t TStringPacker<TStringType>::MeasureLeaf(const TStringType& str) const {
        size_t len = str.size(); 
        return TIntegralPacker<size_t>().MeasureLeaf(len) + len * sizeof(typename TStringType::char_type);
    } 
 
    template <class TStringType>
    inline size_t TStringPacker<TStringType>::SkipLeaf(const char* buf) const {
        size_t result = TIntegralPacker<size_t>().SkipLeaf(buf);
        {
            size_t len;
            TIntegralPacker<size_t>().UnpackLeaf(buf, len);
            result += len * sizeof(typename TStringType::char_type);
        }
        return result;
    } 
 
    template <class T>
    class TPacker;

    // TContainerPacker --- for any container
    // Requirements to class C:
    //    - has method size() (returns size_t)
    //    - has subclass C::value_type
    //    - has subclass C::const_iterator
    //    - has methods begin() and end() (return C::const_iterator)
    //    - has method insert(C::const_iterator, const C::value_type&)
    //  Examples: TVector, TList, TSet
    //  Requirements to class EP: has methods as in any packer (UnpackLeaf, PackLeaf, MeasureLeaf, SkipLeaf) that
    //    are applicable to C::value_type

    template <typename T>
    struct TContainerInfo {
        enum {
            IsVector = 0
        };
    };

    template <typename T>
    struct TContainerInfo<std::vector<T>> {
        enum {
            IsVector = 1
        };
    };

    template <typename T>
    struct TContainerInfo<TVector<T>> {
        enum {
            IsVector = 1
        };
    };

    template <bool IsVector>
    class TContainerPackerHelper {
    };

    template <>
    class TContainerPackerHelper<false> {
    public:
        template <class Packer, class Container>
        static void UnpackLeaf(Packer& p, const char* buffer, Container& c) {
            p.UnpackLeafSimple(buffer, c);
        }
    };

    template <>
    class TContainerPackerHelper<true> {
    public:
        template <class Packer, class Container>
        static void UnpackLeaf(Packer& p, const char* buffer, Container& c) {
            p.UnpackLeafVector(buffer, c);
        }
    };

    template <class C, class EP = TPacker<typename C::value_type>>
    class TContainerPacker {
    private:
        typedef C TContainer;
        typedef EP TElementPacker;
        typedef typename TContainer::const_iterator TElementIterator;

        void UnpackLeafSimple(const char* buffer, TContainer& c) const;
        void UnpackLeafVector(const char* buffer, TContainer& c) const;

        friend class TContainerPackerHelper<TContainerInfo<C>::IsVector>;

    public:
        void UnpackLeaf(const char* buffer, TContainer& c) const {
            TContainerPackerHelper<TContainerInfo<C>::IsVector>::UnpackLeaf(*this, buffer, c);
        }
        void PackLeaf(char* buffer, const TContainer& data, size_t size) const;
        size_t MeasureLeaf(const TContainer& data) const;
        size_t SkipLeaf(const char* buffer) const;
    };

    template <class C, class EP>
    inline void TContainerPacker<C, EP>::UnpackLeafSimple(const char* buffer, C& result) const {
        size_t offset = TIntegralPacker<size_t>().SkipLeaf(buffer); // first value is the total size (not needed here)
        size_t len;
        TIntegralPacker<size_t>().UnpackLeaf(buffer + offset, len);
        offset += TIntegralPacker<size_t>().SkipLeaf(buffer + offset);

        result.clear();

        typename C::value_type value;
        for (size_t i = 0; i < len; i++) {
            TElementPacker().UnpackLeaf(buffer + offset, value);
            result.insert(result.end(), value);
            offset += TElementPacker().SkipLeaf(buffer + offset);
        }
    }

    template <class C, class EP>
    inline void TContainerPacker<C, EP>::UnpackLeafVector(const char* buffer, C& result) const {
        size_t offset = TIntegralPacker<size_t>().SkipLeaf(buffer); // first value is the total size (not needed here)
        size_t len;
        TIntegralPacker<size_t>().UnpackLeaf(buffer + offset, len);
        offset += TIntegralPacker<size_t>().SkipLeaf(buffer + offset);
        result.resize(len);

        for (size_t i = 0; i < len; i++) {
            TElementPacker().UnpackLeaf(buffer + offset, result[i]);
            offset += TElementPacker().SkipLeaf(buffer + offset);
        }
    }

    template <class C, class EP>
    inline void TContainerPacker<C, EP>::PackLeaf(char* buffer, const C& data, size_t size) const {
        size_t sizeOfSize = TIntegralPacker<size_t>().MeasureLeaf(size);
        TIntegralPacker<size_t>().PackLeaf(buffer, size, sizeOfSize);
        size_t len = data.size();
        size_t curSize = TIntegralPacker<size_t>().MeasureLeaf(len);
        TIntegralPacker<size_t>().PackLeaf(buffer + sizeOfSize, len, curSize);
        curSize += sizeOfSize;
        for (TElementIterator p = data.begin(); p != data.end(); p++) {
            size_t sizeChange = TElementPacker().MeasureLeaf(*p);
            TElementPacker().PackLeaf(buffer + curSize, *p, sizeChange);
            curSize += sizeChange;
        }
        Y_ASSERT(curSize == size);
    }

    template <class C, class EP>
    inline size_t TContainerPacker<C, EP>::MeasureLeaf(const C& data) const {
        size_t curSize = TIntegralPacker<size_t>().MeasureLeaf(data.size());
        for (TElementIterator p = data.begin(); p != data.end(); p++)
            curSize += TElementPacker().MeasureLeaf(*p);
        size_t extraSize = TIntegralPacker<size_t>().MeasureLeaf(curSize);

        // Double measurement protects against sudden increases in extraSize,
        // e.g. when curSize is 127 and stays in one byte, but curSize + 1 requires two bytes.

        extraSize = TIntegralPacker<size_t>().MeasureLeaf(curSize + extraSize);
        Y_ASSERT(extraSize == TIntegralPacker<size_t>().MeasureLeaf(curSize + extraSize));
        return curSize + extraSize;
    }

    template <class C, class EP>
    inline size_t TContainerPacker<C, EP>::SkipLeaf(const char* buffer) const {
        size_t value;
        TIntegralPacker<size_t>().UnpackLeaf(buffer, value);
        return value;
    }

    // TPairPacker --- for std::pair<T1, T2> (any two types; can be nested)
    // TPacker<T1> and TPacker<T2> should be valid classes

    template <class T1, class T2, class TPacker1 = TPacker<T1>, class TPacker2 = TPacker<T2>>
    class TPairPacker {
    private:
        typedef std::pair<T1, T2> TMyPair;

    public:
        void UnpackLeaf(const char* buffer, TMyPair& pair) const;
        void PackLeaf(char* buffer, const TMyPair& data, size_t size) const;
        size_t MeasureLeaf(const TMyPair& data) const;
        size_t SkipLeaf(const char* buffer) const;
    };

    template <class T1, class T2, class TPacker1, class TPacker2>
    inline void TPairPacker<T1, T2, TPacker1, TPacker2>::UnpackLeaf(const char* buffer, std::pair<T1, T2>& pair) const {
        TPacker1().UnpackLeaf(buffer, pair.first);
        size_t size = TPacker1().SkipLeaf(buffer);
        TPacker2().UnpackLeaf(buffer + size, pair.second);
    }

    template <class T1, class T2, class TPacker1, class TPacker2>
    inline void TPairPacker<T1, T2, TPacker1, TPacker2>::PackLeaf(char* buffer, const std::pair<T1, T2>& data, size_t size) const {
        size_t size1 = TPacker1().MeasureLeaf(data.first);
        TPacker1().PackLeaf(buffer, data.first, size1);
        size_t size2 = TPacker2().MeasureLeaf(data.second);
        TPacker2().PackLeaf(buffer + size1, data.second, size2);
        Y_ASSERT(size == size1 + size2);
    }

    template <class T1, class T2, class TPacker1, class TPacker2>
    inline size_t TPairPacker<T1, T2, TPacker1, TPacker2>::MeasureLeaf(const std::pair<T1, T2>& data) const {
        size_t size1 = TPacker1().MeasureLeaf(data.first);
        size_t size2 = TPacker2().MeasureLeaf(data.second);
        return size1 + size2;
    }

    template <class T1, class T2, class TPacker1, class TPacker2>
    inline size_t TPairPacker<T1, T2, TPacker1, TPacker2>::SkipLeaf(const char* buffer) const {
        size_t size1 = TPacker1().SkipLeaf(buffer);
        size_t size2 = TPacker2().SkipLeaf(buffer + size1);
        return size1 + size2;
    }

    //------------------------------------------------------------------------------------------
    // Packer for fixed-size arrays, i.e. for std::array.
    // Saves memory by not storing anything about their size.
    // SkipLeaf skips every value, so can be slow for big arrays.
    // Requires std::tuple_size<TValue>, TValue::operator[] and possibly TValue::value_type.
    template <class TValue, class TElementPacker = TPacker<typename TValue::value_type>>
    class TArrayPacker {
    public:
        using TElemPacker = TElementPacker;

        enum {
            Size = std::tuple_size<TValue>::value
        };

        void UnpackLeaf(const char* p, TValue& t) const {
            const char* buf = p;
            for (size_t i = 0; i < Size; ++i) {
                TElemPacker().UnpackLeaf(buf, t[i]);
                buf += TElemPacker().SkipLeaf(buf);
            }
        }

        void PackLeaf(char* buffer, const TValue& data, size_t computedSize) const {
            size_t remainingSize = computedSize;
            char* pos = buffer;
            for (size_t i = 0; i < Size; ++i) {
                const size_t elemSize = TElemPacker().MeasureLeaf(data[i]);
                TElemPacker().PackLeaf(pos, data[i], Min(elemSize, remainingSize));
                pos += elemSize;
                remainingSize -= elemSize;
            }
        }

        size_t MeasureLeaf(const TValue& data) const {
            size_t result = 0;
            for (size_t i = 0; i < Size; ++i) {
                result += TElemPacker().MeasureLeaf(data[i]);
            }
            return result;
        }

        size_t SkipLeaf(const char* p) const // this function better be fast because it is very frequently used
        {
            const char* buf = p;
            for (size_t i = 0; i < Size; ++i) {
                buf += TElemPacker().SkipLeaf(buf);
            }
            return buf - p;
        }
    };

    //------------------------------------
    // TPacker --- the generic packer.

    template <class T, bool IsIntegral>
    class TPackerImpl;

    template <class T> 
    class TPackerImpl<T, true>: public TIntegralPacker<T> {
    };
    // No implementation for non-integral types.
 
    template <class T> 
    class TPacker: public TPackerImpl<T, std::is_integral<T>::value> {
    };
 
    template <>
    class TPacker<float>: public TAsIsPacker<float> {
    };
 
    template <>
    class TPacker<double>: public TAsIsPacker<double> {
    };
 
    template <>
    class TPacker<TString>: public TStringPacker<TString> {
    };

    template <>
    class TPacker<TUtf16String>: public TStringPacker<TUtf16String> {
    }; 
 
    template <> 
    class TPacker<TStringBuf>: public TStringPacker<TStringBuf> {
    };

    template <> 
    class TPacker<TWtringBuf>: public TStringPacker<TWtringBuf> {
    }; 

    template <class T>
    class TPacker<std::vector<T>>: public TContainerPacker<std::vector<T>> {
    };

    template <class T>
    class TPacker<TVector<T>>: public TContainerPacker<TVector<T>> {
    };

    template <class T>
    class TPacker<std::list<T>>: public TContainerPacker<std::list<T>> {
    };

    template <class T>
    class TPacker<TList<T>>: public TContainerPacker<TList<T>> {
    };

    template <class T>
    class TPacker<std::set<T>>: public TContainerPacker<std::set<T>> {
    };

    template <class T>
    class TPacker<TSet<T>>: public TContainerPacker<TSet<T>> {
    };

    template <class T1, class T2>
    class TPacker<std::pair<T1, T2>>: public TPairPacker<T1, T2> {
    };

    template <class T, size_t N>
    class TPacker<std::array<T, N>>: public TArrayPacker<std::array<T, N>> {
    };

}