#ifndef OSRM_STORAGE_SERIALIZATION_HPP #define OSRM_STORAGE_SERIALIZATION_HPP #include "util/deallocating_vector.hpp" #include "util/integer_range.hpp" #include "util/vector_view.hpp" #include "storage/io.hpp" #include "storage/shared_datatype.hpp" #include "storage/tar.hpp" #include #include #include #include #include #include #if USE_STXXL_LIBRARY #include #endif namespace osrm { namespace storage { namespace serialization { namespace detail { template inline BlockT packBits(const T &data, std::size_t base_index, const std::size_t count) { static_assert(std::is_same::value, "value_type is not bool"); static_assert(std::is_unsigned::value, "BlockT must be unsigned type"); static_assert(std::is_integral::value, "BlockT must be an integral type"); static_assert(CHAR_BIT == 8, "Non-8-bit bytes not supported, sorry!"); BOOST_ASSERT(sizeof(BlockT) * CHAR_BIT >= count); // Note: if this packing is changed, be sure to update vector_view // as well, so that on-disk and in-memory layouts match. BlockT value = 0; for (std::size_t bit = 0; bit < count; ++bit) { value |= (data[base_index + bit] ? BlockT{1} : BlockT{0}) << bit; } return value; } template inline void unpackBits(T &data, const std::size_t base_index, const std::size_t count, const BlockT value) { static_assert(std::is_same::value, "value_type is not bool"); static_assert(std::is_unsigned::value, "BlockT must be unsigned type"); static_assert(std::is_integral::value, "BlockT must be an integral type"); static_assert(CHAR_BIT == 8, "Non-8-bit bytes not supported, sorry!"); BOOST_ASSERT(sizeof(BlockT) * CHAR_BIT >= count); for (std::size_t bit = 0; bit < count; ++bit) { data[base_index + bit] = value & (BlockT{1} << bit); } } template void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data) { const auto count = reader.ReadElementCount64(name); data.resize(count); std::uint64_t index = 0; using BlockType = std::uint64_t; constexpr std::uint64_t BLOCK_BITS = CHAR_BIT * sizeof(BlockType); const auto decode = [&](const BlockType block) { auto read_size = std::min(count - index, BLOCK_BITS); unpackBits(data, index, read_size, block); index += BLOCK_BITS; }; reader.ReadStreaming(name, boost::make_function_output_iterator(decode)); } template void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data) { const auto count = data.size(); writer.WriteElementCount64(name, count); std::uint64_t index = 0; using BlockType = std::uint64_t; constexpr std::uint64_t BLOCK_BITS = CHAR_BIT * sizeof(BlockType); // FIXME on old boost version the function_input_iterator does not work with lambdas // so we need to wrap it in a function here. const std::function encode_function = [&]() -> BlockType { auto write_size = std::min(count - index, BLOCK_BITS); auto packed = packBits(data, index, write_size); index += BLOCK_BITS; return packed; }; std::uint64_t number_of_blocks = (count + BLOCK_BITS - 1) / BLOCK_BITS; writer.WriteStreaming( name, boost::make_function_input_iterator(encode_function, boost::infinite()), number_of_blocks); } } /* All vector formats here use the same on-disk format. * This is important because we want to be able to write from a vector * of one kind, but read it into a vector of another kind. * * All vector types with this guarantee should be placed in this file. */ template inline void read(storage::tar::FileReader &reader, const std::string &name, util::DeallocatingVector &vec) { vec.resize(reader.ReadElementCount64(name)); reader.ReadStreaming(name, vec.begin(), vec.size()); } template inline void write(storage::tar::FileWriter &writer, const std::string &name, const util::DeallocatingVector &vec) { writer.WriteElementCount64(name, vec.size()); writer.WriteStreaming(name, vec.begin(), vec.size()); } #if USE_STXXL_LIBRARY template inline void read(storage::tar::FileReader &reader, const std::string &name, stxxl::vector &vec) { auto size = reader.ReadElementCount64(name); vec.reserve(size); reader.ReadStreaming(name, std::back_inserter(vec), size); } template inline void write(storage::tar::FileWriter &writer, const std::string &name, const stxxl::vector &vec) { writer.WriteElementCount64(name, vec.size()); writer.WriteStreaming(name, vec.begin(), vec.size()); } #endif template void read(io::BufferReader &reader, std::vector &data) { const auto count = reader.ReadElementCount64(); data.resize(count); reader.ReadInto(data.data(), count); } template void write(io::BufferWriter &writer, const std::vector &data) { const auto count = data.size(); writer.WriteElementCount64(count); writer.WriteFrom(data.data(), count); } template inline void write(io::BufferWriter &writer, const T &data) { writer.WriteFrom(data); } template inline void read(io::BufferReader &reader, T &data) { reader.ReadInto(data); } inline void write(io::BufferWriter &writer, const std::string &data) { const auto count = data.size(); writer.WriteElementCount64(count); writer.WriteFrom(data.data(), count); } inline void read(io::BufferReader &reader, std::string &data) { const auto count = reader.ReadElementCount64(); data.resize(count); reader.ReadInto(const_cast(data.data()), count); } inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data) { const auto count = data.size(); writer.WriteElementCount64(name, count); writer.WriteFrom(name, data.data(), count); } inline void read(tar::FileReader &reader, const std::string &name, std::string &data) { const auto count = reader.ReadElementCount64(name); data.resize(count); reader.ReadInto(name, const_cast(data.data()), count); } template inline void read(tar::FileReader &reader, const std::string &name, std::vector &data) { const auto count = reader.ReadElementCount64(name); data.resize(count); reader.ReadInto(name, data.data(), count); } template void write(tar::FileWriter &writer, const std::string &name, const std::vector &data) { const auto count = data.size(); writer.WriteElementCount64(name, count); writer.WriteFrom(name, data.data(), count); } template void read(tar::FileReader &reader, const std::string &name, util::vector_view &data) { const auto count = reader.ReadElementCount64(name); data.resize(count); reader.ReadInto(name, data.data(), count); } template void write(tar::FileWriter &writer, const std::string &name, const util::vector_view &data) { const auto count = data.size(); writer.WriteElementCount64(name, count); writer.WriteFrom(name, data.data(), count); } template <> inline void read(tar::FileReader &reader, const std::string &name, util::vector_view &data) { detail::readBoolVector(reader, name, data); } template <> inline void write(tar::FileWriter &writer, const std::string &name, const util::vector_view &data) { detail::writeBoolVector(writer, name, data); } template <> inline void read(tar::FileReader &reader, const std::string &name, std::vector &data) { detail::readBoolVector(reader, name, data); } template <> inline void write(tar::FileWriter &writer, const std::string &name, const std::vector &data) { detail::writeBoolVector(writer, name, data); } template void read(io::BufferReader &reader, std::map &data) { const auto count = reader.ReadElementCount64(); for (auto index : util::irange(0, count)) { (void)index; std::pair pair; read(reader, pair.first); read(reader, pair.second); data.insert(pair); } } template void write(io::BufferWriter &writer, const std::map &data) { const auto count = data.size(); writer.WriteElementCount64(count); for (const auto &pair : data) { write(writer, pair.first); write(writer, pair.second); } } inline void read(io::BufferReader &reader, BaseDataLayout &layout) { read(reader, layout.blocks); } inline void write(io::BufferWriter &writer, const BaseDataLayout &layout) { write(writer, layout.blocks); } } } } #endif