Converting reader definition to be able to load half-grids from stored float-grids

Author: konivo

openvdb/openvdb/io/Archive.cc

@@ -212,6 +212,8 @@ struct StreamMetadata::Impl
     bool mDelayedLoadMeta = DelayedLoadMetadata::isRegisteredType();
     uint64_t mLeaf = 0;
     uint32_t mTest = 0; // for testing only
+    std::string mDesiredScalarType = "";
+    SharedPtr<ConvertingReaderBase> mConvertingReader;
 }; // struct StreamMetadata
 
 
@@ -279,6 +281,12 @@ MetaMap&        StreamMetadata::gridMetadata()          { return mImpl->mGridMet
 const MetaMap&  StreamMetadata::gridMetadata() const    { return mImpl->mGridMetadata; }
 uint32_t        StreamMetadata::__test() const          { return mImpl->mTest; }
 
+const std::string&    StreamMetadata::desiredScalarType() const { return mImpl->mDesiredScalarType; }
+void StreamMetadata::setDesiredScalarType(std::string t) { mImpl->mDesiredScalarType = t; }
+
+const ConvertingReaderBase* StreamMetadata::convertingReader() const {return mImpl->mConvertingReader.get();}
+void StreamMetadata::setConvertingReader(SharedPtr<ConvertingReaderBase> t) {mImpl->mConvertingReader = t;}
+
 StreamMetadata::AuxDataMap& StreamMetadata::auxData() { return mImpl->mAuxData; }
 const StreamMetadata::AuxDataMap& StreamMetadata::auxData() const { return mImpl->mAuxData; }
 
@@ -1186,6 +1194,17 @@ Archive::isDelayedLoadingEnabled()
 }
 
 
+Name Archive::conversionToString(Archive::ScalarConversion conv)
+{
+    switch (conv)
+    {
+        case Archive::ScalarConversion::None:
+            return "";
+        case Archive::ScalarConversion::Half:
+            return typeNameAsString<math::half>();
+    }
+}
+
 namespace {
 
 struct NoBBox {};
@@ -1223,6 +1242,7 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const
         streamMetadata.reset(new StreamMetadata);
     }
     streamMetadata->setHalfFloat(grid->saveFloatAsHalf());
+    streamMetadata->setConvertingReader(gd.convertingReader());
     io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false);
 
     io::setGridClass(is, GRID_UNKNOWN);

openvdb/openvdb/io/Archive.h

@@ -34,6 +34,8 @@ class OPENVDB_API Archive
     using Ptr = SharedPtr<Archive>;
     using ConstPtr = SharedPtr<const Archive>;
 
+    enum class ScalarConversion { None, Half };
+
     static const uint32_t DEFAULT_COMPRESSION_FLAGS;
 
     Archive();
@@ -98,6 +100,8 @@ class OPENVDB_API Archive
     /// to disable delayed loading unconditionally.
     static bool isDelayedLoadingEnabled();
 
+    static Name conversionToString(ScalarConversion conv);
+
 protected:
     /// @brief Return @c true if the input stream contains grid offsets
     /// that allow for random access or partial reading.

openvdb/openvdb/io/Compression.h

@@ -320,6 +320,197 @@ struct HalfReader</*IsReal=*/true, T> {
 };
 
 
+struct ConvertingReaderBase
+{
+    using Ptr = SharedPtr<ConvertingReaderBase>;
+
+    virtual ~ConvertingReaderBase() = default;
+};
+
+template<typename ValueT>
+struct ConvertingReader: ConvertingReaderBase
+{
+    virtual void read(
+        std::istream& is, ValueT* data, Index count, uint32_t compression,
+        DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0),
+        bool fromHalf = false) const = 0;
+
+    virtual void read(std::istream& is, ValueT& data) const = 0;
+
+    virtual void seekElement(std::istream& is, int offset, std::ios_base::seekdir dir) const = 0;
+
+    static const ConvertingReader& get(std::istream& is);
+};
+
+template<typename ValueT, typename TValueFrom>
+struct TypedConvertingReader: ConvertingReader<ValueT>
+{
+    using HalfT = typename RealToHalf<TValueFrom>::HalfT;
+
+    static constexpr bool do_conversion                 = !std::is_same<ValueT, TValueFrom>::value;
+    static constexpr bool from_half_flag_is_relevant    = RealToHalf<TValueFrom>::isReal;
+    
+    template<typename TRead>
+    static void readHelper(
+        std::istream& is, ValueT* data, Index count, uint32_t compression,
+        DelayedLoadMetadata* metadata, size_t metadataOffset)
+    {
+        if constexpr (std::is_same<ValueT, TRead>::value)
+        {
+            // either reads or skips through compressed data (count == 0) or it
+            // is seeking (data == nullptr)
+            io::readData(is, data, count, compression, metadata, metadataOffset);
+        }
+        else
+        {
+            // reading
+            if (data && count > 0)
+            {
+                std::vector<TRead> buffer(count);
+                io::readData(is, buffer.data(), count, compression, metadata, metadataOffset);
+                std::copy(buffer.begin(), buffer.end(), data);
+            }
+            // either skips through compressed data (count == 0) or it is
+            // seeking (data == nullptr)
+            else
+            {
+                // reinterpret_cast<TRead*>(data) might produce not-aligned
+                // pointer, but io::readData will not write to it, so it should
+                // be safe. We must not skip this statement, nor we can pass
+                // nullptr instead of data, it causes failures in the testsuite.
+                // Now it behaves exactly as before.
+                io::readData(is, reinterpret_cast<TRead*>(data), count,
+                    compression, metadata, metadataOffset);
+            }
+        }
+    }
+
+    virtual void read(
+        std::istream& is, ValueT* data, Index count, uint32_t compression,
+        DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0),
+        bool fromHalf = false) const override
+    {
+        // this if-statement is here to maintain the original logic when
+        // HalfReader<T> is being used. It can't be removed as it breaks bunch
+        // of tests in the testsuite.
+        if (fromHalf && from_half_flag_is_relevant && count < 1)
+            return;
+
+        // due to std::vector<bool> being weird and not fitting the overall
+        // templating idea here (it doesn't have data() method), the following
+        // will fail to do non-identity conversion for bool-typed data. But for
+        // bool-typed data there is just identity at the moment.
+        if constexpr (from_half_flag_is_relevant)
+        {
+            if (fromHalf)
+            {
+                readHelper<HalfT>(is, data, count, compression, metadata, metadataOffset);
+            }
+            else
+            {
+                readHelper<TValueFrom>(is, data, count, compression, metadata, metadataOffset);
+            }
+        }
+        else
+        {
+            readHelper<TValueFrom>(is, data, count, compression, metadata, metadataOffset);
+        }
+    }
+
+    virtual void read(std::istream& is, ValueT& data) const override
+    {
+        if constexpr (do_conversion)
+        {
+            TValueFrom buffer;
+            is.read(reinterpret_cast<char*>(&buffer), /*bytes=*/sizeof(TValueFrom));
+            data = buffer;
+        }
+        else
+        {
+            is.read(reinterpret_cast<char*>(&data), /*bytes=*/sizeof(ValueT));
+        }
+    }
+
+    virtual void seekElement(std::istream& is, int offset, std::ios_base::seekdir dir) const override
+    {
+        if constexpr (do_conversion)
+        {
+            is.seekg(/*bytes=*/sizeof(TValueFrom) * offset, dir);
+        }
+        else
+        {
+            is.seekg(/*bytes=*/sizeof(ValueT) * offset, dir);
+        }
+    }
+};
+
+template<typename ValueT>
+inline const ConvertingReader<ValueT>& ConvertingReader<ValueT>::get(std::istream& is)
+{
+    SharedPtr<io::StreamMetadata> meta = io::getStreamMetadataPtr(is);
+
+    auto ptr = static_cast<const ConvertingReader*>(meta->convertingReader());
+    if(ptr)
+    {
+        return *ptr;
+    }
+    auto ptr2 = new TypedConvertingReader<ValueT, ValueT>();
+    meta->setConvertingReader(Ptr(ptr2));
+    return *ptr2;
+}
+
+struct ConvertingReaderFactory
+{
+    using ReaderPtr = ConvertingReaderBase::Ptr;
+
+    struct FactoryEntry
+    {
+        ReaderPtr   reader;
+        Name        new_grid_value_type;
+    };
+
+private:
+    template<typename T1, typename T2>
+    static Name getTypeName()
+    {
+        return Name(typeNameAsString<T1>()) + Name(typeNameAsString<T2>());
+    }
+
+    template<typename T1, typename T2, typename T3>
+    static auto getMapEntry()
+    {
+        return std::pair
+        { 
+            getTypeName<T1, T2>(),
+            FactoryEntry { ReaderPtr(new TypedConvertingReader<T3, T1>()), typeNameAsString<T3>()}
+        };
+    }
+
+public:
+    static FactoryEntry create(const Name& grid_value_type, const Name& desired_scalar_type)
+    {
+        static std::unordered_map<Name, FactoryEntry> type_map =
+        {
+            getMapEntry<float, Half, Half>(),
+            getMapEntry<double, Half, Half>(),
+            getMapEntry<Vec2f, Half, Vec2H>(),
+            getMapEntry<Vec2d, Half, Vec2H>(),
+            getMapEntry<Vec3f, Half, Vec3H>(),
+            getMapEntry<Vec3d, Half, Vec3H>(),
+        };
+
+        if (auto it = type_map.find(grid_value_type + desired_scalar_type); it != type_map.end())
+        {
+            return it->second;
+        }
+        else
+        {
+            return {nullptr, ""};
+        }
+    }
+};
+
+
 template<typename T>
 inline size_t
 writeDataSize(const T *data, Index count, uint32_t compression)
@@ -474,6 +665,9 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
     const bool seek = (destBuf == nullptr);
     OPENVDB_ASSERT(!seek || (!meta || meta->seekable()));
 
+    // converting reader for reading grid values
+    auto& convertingReader = io::ConvertingReader<ValueT>::get(is);
+
     // Get delayed load metadata if it exists
 
     DelayedLoadMetadata::Ptr delayLoadMeta;
@@ -512,16 +706,16 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
     {
         // Read one of at most two distinct inactive values.
         if (seek) {
-            is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
+            convertingReader.seekElement(is, 1, std::ios_base::cur);
         } else {
-            is.read(reinterpret_cast<char*>(&inactiveVal0), /*bytes=*/sizeof(ValueT));
+            convertingReader.read(is, inactiveVal0);
         }
         if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
             // Read the second of two distinct inactive values.
             if (seek) {
-                is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
+                convertingReader.seekElement(is, 1, std::ios_base::cur);
             } else {
-                is.read(reinterpret_cast<char*>(&inactiveVal1), /*bytes=*/sizeof(ValueT));
+                convertingReader.read(is, inactiveVal1);
             }
         }
     }
@@ -558,13 +752,8 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
     }
 
     // Read in the buffer.
-    if (fromHalf) {
-        HalfReader<RealToHalf<ValueT>::isReal, ValueT>::read(
-            is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
-    } else {
-        readData<ValueT>(
-            is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
-    }
+    convertingReader.read(
+        is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex, fromHalf);
 
     // If mask compression is enabled and the number of active values read into
     // the temp buffer is smaller than the size of the destination buffer,

openvdb/openvdb/io/File.cc

@@ -280,9 +280,9 @@ File::isOpen() const
 
 bool
 #ifdef OPENVDB_USE_DELAYED_LOADING
-File::open(bool delayLoad, const MappedFile::Notifier& notifier)
+File::open(bool delayLoad, const MappedFile::Notifier& notifier, ScalarConversion conversion)
 #else
-File::open(bool /*delayLoad = true*/)
+File::open(bool /*delayLoad = true*/, ScalarConversion conversion)
 #endif // OPENVDB_USE_DELAYED_LOADING
 {
     if (isOpen()) {
@@ -367,6 +367,7 @@ File::open(bool /*delayLoad = true*/)
     // and other metadata.
     mImpl->mStreamMetadata.reset(new StreamMetadata);
     mImpl->mStreamMetadata->setSeekable(true);
+    mImpl->mStreamMetadata->setDesiredScalarType(conversionToString(conversion));
     io::setStreamMetadataPtr(inputStream(), mImpl->mStreamMetadata, /*transfer=*/false);
     Archive::setFormatVersion(inputStream());
     Archive::setLibraryVersion(inputStream());

openvdb/openvdb/io/File.h

@@ -64,9 +64,10 @@ class OPENVDB_API File: public Archive
     /// @throw IoError if the file is not a valid VDB file.
     /// @return @c true if the file's UUID has changed since it was last read.
     /// @see setCopyMaxBytes
-    bool open(bool delayLoad = true, const MappedFile::Notifier& = MappedFile::Notifier());
+    bool open(bool delayLoad = true, const MappedFile::Notifier& = MappedFile::Notifier(),
+        ScalarConversion conversion = ScalarConversion::None);
 #else
-    bool open(bool /*delayLoad*/ = false);
+    bool open(bool /*delayLoad*/ = false, ScalarConversion conversion = ScalarConversion::None);
 #endif
 
     /// Return @c true if the file has been opened for reading.

openvdb/openvdb/io/GridDescriptor.cc

@@ -5,7 +5,9 @@
 
 #include <openvdb/Exceptions.h>
 #include <openvdb/util/Name.h>
+#include <openvdb/io/Compression.h>
 #include <sstream>
+#include <iostream>
 
 
 namespace openvdb {
@@ -94,6 +96,26 @@ GridDescriptor::read(std::istream &is)
     }
     // else
     GridBase::Ptr grid = GridBase::createGrid(mGridType);
+
+    // change the type of grid if it is desired and possible
+    Name desired_scalar_type = io::getStreamMetadataPtr(is)->desiredScalarType();
+    Name grid_value_type = grid->valueType();
+    
+    auto [reader, new_value_type] = io::ConvertingReaderFactory::create(grid_value_type, desired_scalar_type);
+    if (reader)
+    {
+        Name newGridType = mGridType;
+        newGridType.replace(newGridType.find(grid_value_type), grid_value_type.length(), new_value_type);
+
+        // Create the grid of the type if it has been registered.
+        if (GridBase::isRegistered(newGridType)) {
+            mGridType = newGridType;
+            mConvertingReader = reader;
+            grid = GridBase::createGrid(mGridType);
+        }
+    }
+
+    // TODO: store reader into grid/gd
     if (grid) grid->setSaveFloatAsHalf(mSaveFloatAsHalf);
 
     // Read in the offsets.