OpenVDB: openvdb/io/Compression.h Source File

OpenVDB 12.1.1

Loading...
Searching...
No Matches
Compression.h
Go to the documentation of this file.
1// Copyright Contributors to the OpenVDB Project
2// SPDX-License-Identifier: Apache-2.0
3
4#ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
5#define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
6
7#include <openvdb/Types.h>
8#include <openvdb/MetaMap.h>
9#include <openvdb/math/Math.h> // for negative()
10#include <openvdb/util/Assert.h>
11#include "io.h" // for getDataCompression(), etc.
12#include "DelayedLoadMetadata.h"
13#include <algorithm>
14#include <iostream>
15#include <memory>
16#include <string>
17#include <vector>
18
19
20namespace openvdb {
21 OPENVDB_USE_VERSION_NAMESPACE
22namespace OPENVDB_VERSION_NAME {
23namespace io {
24
25/// @brief OR-able bit flags for compression options on input and output streams
26/// @details
27/// <dl>
28/// <dt><tt>COMPRESS_NONE</tt>
29/// <dd>On write, don't compress data.<br>
30/// On read, the input stream contains uncompressed data.
31///
32/// <dt><tt>COMPRESS_ZIP</tt>
33/// <dd>When writing grids other than level sets or fog volumes, apply
34/// ZLIB compression to internal and leaf node value buffers.<br>
35/// When reading grids other than level sets or fog volumes, indicate that
36/// the value buffers of internal and leaf nodes are ZLIB-compressed.<br>
37/// ZLIB compresses well but is slow.
38///
39/// <dt><tt>COMPRESS_ACTIVE_MASK</tt>
40/// <dd>When writing a grid of any class, don't output a node's inactive values
41/// if it has two or fewer distinct values. Instead, output minimal information
42/// to permit the lossless reconstruction of inactive values.<br>
43/// On read, nodes might have been stored without inactive values.
44/// Where necessary, reconstruct inactive values from available information.
45///
46/// <dt><tt>COMPRESS_BLOSC</tt>
47/// <dd>When writing grids other than level sets or fog volumes, apply
48/// Blosc compression to internal and leaf node value buffers.<br>
49/// When reading grids other than level sets or fog volumes, indicate that
50/// the value buffers of internal and leaf nodes are Blosc-compressed.<br>
51/// Blosc is much faster than ZLIB and produces comparable file sizes.
52/// </dl>
53enum {
54 COMPRESS_NONE = 0,
55 COMPRESS_ZIP = 0x1,
56 COMPRESS_ACTIVE_MASK = 0x2,
57 COMPRESS_BLOSC = 0x4
58};
59
60/// Return a string describing the given compression flags.
61 OPENVDB_API std::string compressionToString(uint32_t flags);
62
63
64////////////////////////////////////////
65
66
67/// @internal Per-node indicator byte that specifies what additional metadata
68/// is stored to permit reconstruction of inactive values
69enum {
70 /*0*/ NO_MASK_OR_INACTIVE_VALS, // no inactive vals, or all inactive vals are +background
71 /*1*/ NO_MASK_AND_MINUS_BG, // all inactive vals are -background
72 /*2*/ NO_MASK_AND_ONE_INACTIVE_VAL, // all inactive vals have the same non-background val
73 /*3*/ MASK_AND_NO_INACTIVE_VALS, // mask selects between -background and +background
74 /*4*/ MASK_AND_ONE_INACTIVE_VAL, // mask selects between backgd and one other inactive val
75 /*5*/ MASK_AND_TWO_INACTIVE_VALS, // mask selects between two non-background inactive vals
76 /*6*/ NO_MASK_AND_ALL_VALS // > 2 inactive vals, so no mask compression at all
77};
78
79
80template <typename ValueT, typename MaskT>
81 struct MaskCompress
82{
83 // Comparison function for values
84 static inline bool eq(const ValueT& a, const ValueT& b) {
85 return math::isExactlyEqual(a, b);
86 }
87
88 MaskCompress(
89 const MaskT& valueMask, const MaskT& childMask,
90 const ValueT* srcBuf, const ValueT& background)
91 {
92 /// @todo Consider all values, not just inactive values?
93 inactiveVal[0] = inactiveVal[1] = background;
94 int numUniqueInactiveVals = 0;
95 for (typename MaskT::OffIterator it = valueMask.beginOff();
96 numUniqueInactiveVals < 3 && it; ++it)
97 {
98 const Index32 idx = it.pos();
99
100 // Skip inactive values that are actually child node pointers.
101 if (childMask.isOn(idx)) continue;
102
103 const ValueT& val = srcBuf[idx];
104 const bool unique = !(
105 (numUniqueInactiveVals > 0 && MaskCompress::eq(val, inactiveVal[0])) ||
106 (numUniqueInactiveVals > 1 && MaskCompress::eq(val, inactiveVal[1]))
107 );
108 if (unique) {
109 if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
110 ++numUniqueInactiveVals;
111 }
112 }
113
114 metadata = NO_MASK_OR_INACTIVE_VALS;
115
116 if (numUniqueInactiveVals == 1) {
117 if (!MaskCompress::eq(inactiveVal[0], background)) {
118 if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
119 metadata = NO_MASK_AND_MINUS_BG;
120 } else {
121 metadata = NO_MASK_AND_ONE_INACTIVE_VAL;
122 }
123 }
124 } else if (numUniqueInactiveVals == 2) {
125 metadata = NO_MASK_OR_INACTIVE_VALS;
126 if (!MaskCompress::eq(inactiveVal[0], background) && !MaskCompress::eq(inactiveVal[1], background)) {
127 // If neither inactive value is equal to the background, both values
128 // need to be saved, along with a mask that selects between them.
129 metadata = MASK_AND_TWO_INACTIVE_VALS;
130
131 } else if (MaskCompress::eq(inactiveVal[1], background)) {
132 if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
133 // If the second inactive value is equal to the background and
134 // the first is equal to -background, neither value needs to be saved,
135 // but save a mask that selects between -background and +background.
136 metadata = MASK_AND_NO_INACTIVE_VALS;
137 } else {
138 // If the second inactive value is equal to the background, only
139 // the first value needs to be saved, along with a mask that selects
140 // between it and the background.
141 metadata = MASK_AND_ONE_INACTIVE_VAL;
142 }
143 } else if (MaskCompress::eq(inactiveVal[0], background)) {
144 if (MaskCompress::eq(inactiveVal[1], math::negative(background))) {
145 // If the first inactive value is equal to the background and
146 // the second is equal to -background, neither value needs to be saved,
147 // but save a mask that selects between -background and +background.
148 metadata = MASK_AND_NO_INACTIVE_VALS;
149 std::swap(inactiveVal[0], inactiveVal[1]);
150 } else {
151 // If the first inactive value is equal to the background, swap it
152 // with the second value and save only that value, along with a mask
153 // that selects between it and the background.
154 std::swap(inactiveVal[0], inactiveVal[1]);
155 metadata = MASK_AND_ONE_INACTIVE_VAL;
156 }
157 }
158 } else if (numUniqueInactiveVals > 2) {
159 metadata = NO_MASK_AND_ALL_VALS;
160 }
161 }
162
163 int8_t metadata = NO_MASK_AND_ALL_VALS;
164 ValueT inactiveVal[2];
165};
166
167
168////////////////////////////////////////
169
170
171/// @brief RealToHalf and its specializations define a mapping from
172/// floating-point data types to analogous half float types.
173template<typename T>
174 struct RealToHalf {
175 enum { isReal = false }; // unless otherwise specified, type T is not a floating-point type
176 using HalfT = T; // type T's half float analogue is T itself
177 static HalfT convert(const T& val) { return val; }
178};
179 template<> struct RealToHalf<float> {
180 enum { isReal = true };
181 using HalfT = math::half;
182 static HalfT convert(float val) { return HalfT(val); }
183};
184 template<> struct RealToHalf<double> {
185 enum { isReal = true };
186 using HalfT = math::half;
187 // A half can only be constructed from a float, so cast the value to a float first.
188 static HalfT convert(double val) { return HalfT(float(val)); }
189};
190 template<> struct RealToHalf<Vec2s> {
191 enum { isReal = true };
192 using HalfT = Vec2H;
193 static HalfT convert(const Vec2s& val) { return HalfT(val); }
194};
195 template<> struct RealToHalf<Vec2d> {
196 enum { isReal = true };
197 using HalfT = Vec2H;
198 // A half can only be constructed from a float, so cast the vector's elements to floats first.
199 static HalfT convert(const Vec2d& val) { return HalfT(Vec2s(val)); }
200};
201 template<> struct RealToHalf<Vec3s> {
202 enum { isReal = true };
203 using HalfT = Vec3H;
204 static HalfT convert(const Vec3s& val) { return HalfT(val); }
205};
206 template<> struct RealToHalf<Vec3d> {
207 enum { isReal = true };
208 using HalfT = Vec3H;
209 // A half can only be constructed from a float, so cast the vector's elements to floats first.
210 static HalfT convert(const Vec3d& val) { return HalfT(Vec3s(val)); }
211};
212
213
214/// Return the given value truncated to 16-bit float precision.
215template<typename T>
216inline T
217truncateRealToHalf(const T& val)
218{
219 return T(RealToHalf<T>::convert(val));
220}
221
222
223////////////////////////////////////////
224
225
226 OPENVDB_API size_t zipToStreamSize(const char* data, size_t numBytes);
227 OPENVDB_API void zipToStream(std::ostream&, const char* data, size_t numBytes);
228 OPENVDB_API void unzipFromStream(std::istream&, char* data, size_t numBytes);
229 OPENVDB_API size_t bloscToStreamSize(const char* data, size_t valSize, size_t numVals);
230 OPENVDB_API void bloscToStream(std::ostream&, const char* data, size_t valSize, size_t numVals);
231 OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes);
232
233/// @brief Read data from a stream.
234/// @param is the input stream
235/// @param data the contiguous array of data to read in
236/// @param count the number of elements to read in
237/// @param compression whether and how the data is compressed (either COMPRESS_NONE,
238/// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
239/// @param metadata optional pointer to a DelayedLoadMetadata object that stores
240/// the size of the compressed buffer
241/// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null
242/// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
243/// without Blosc support.
244/// @details This default implementation is instantiated only for types
245/// whose size can be determined by the sizeof() operator.
246template<typename T>
247inline void
248readData(std::istream& is, T* data, Index count, uint32_t compression,
249 DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0))
250{
251 const bool seek = data == nullptr;
252 if (seek) {
253 OPENVDB_ASSERT(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable());
254 }
255 const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP);
256
257 if (metadata && seek && hasCompression) {
258 size_t compressedSize = metadata->getCompressedSize(metadataOffset);
259 is.seekg(compressedSize, std::ios_base::cur);
260 } else if (compression & COMPRESS_BLOSC) {
261 bloscFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
262 } else if (compression & COMPRESS_ZIP) {
263 unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
264 } else if (seek) {
265 is.seekg(sizeof(T) * count, std::ios_base::cur);
266 } else {
267 is.read(reinterpret_cast<char*>(data), sizeof(T) * count);
268 }
269}
270
271/// Specialization for std::string input
272template<>
273 inline void
274 readData<std::string>(std::istream& is, std::string* data, Index count, uint32_t /*compression*/,
275 DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/)
276{
277 for (Index i = 0; i < count; ++i) {
278 size_t len = 0;
279 is >> len;
280 //data[i].resize(len);
281 //is.read(&(data[i][0]), len);
282
283 std::string buffer(len+1, ' ');
284 is.read(&buffer[0], len+1);
285 if (data != nullptr) data[i].assign(buffer, 0, len);
286 }
287}
288
289/// HalfReader wraps a static function, read(), that is analogous to readData(), above,
290/// except that it is partially specialized for floating-point types in order to promote
291/// 16-bit half float values to full float. A wrapper class is required because
292/// only classes, not functions, can be partially specialized.
293 template<bool IsReal, typename T> struct HalfReader;
294/// Partial specialization for non-floating-point types (no half to float promotion)
295template<typename T>
296 struct HalfReader</*IsReal=*/false, T> {
297 static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
298 DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
299 readData(is, data, count, compression, metadata, metadataOffset);
300 }
301};
302/// Partial specialization for floating-point types
303template<typename T>
304 struct HalfReader</*IsReal=*/true, T> {
305 using HalfT = typename RealToHalf<T>::HalfT;
306 static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
307 DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
308 if (count < 1) return;
309 if (data == nullptr) {
310 // seek mode - pass through null pointer
311 readData<HalfT>(is, nullptr, count, compression, metadata, metadataOffset);
312 } else {
313 std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
314 readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compression,
315 metadata, metadataOffset);
316 // Copy half float values from the temporary buffer to the full float output array.
317 std::copy(halfData.begin(), halfData.end(), data);
318 }
319 }
320};
321
322
323template<typename T>
324inline size_t
325writeDataSize(const T *data, Index count, uint32_t compression)
326{
327 if (compression & COMPRESS_BLOSC) {
328 return bloscToStreamSize(reinterpret_cast<const char*>(data), sizeof(T), count);
329 } else if (compression & COMPRESS_ZIP) {
330 return zipToStreamSize(reinterpret_cast<const char*>(data), sizeof(T) * count);
331 } else {
332 return sizeof(T) * count;
333 }
334}
335
336
337/// Specialization for std::string output
338template<>
339 inline size_t
340 writeDataSize<std::string>(const std::string* data, Index count,
341 uint32_t /*compression*/) ///< @todo add compression
342{
343 size_t size(0);
344 for (Index i = 0; i < count; ++i) {
345 const size_t len = data[i].size();
346 size += sizeof(size_t) + (len+1);
347 }
348 return size;
349}
350
351
352/// Write data to a stream.
353/// @param os the output stream
354/// @param data the contiguous array of data to write
355/// @param count the number of elements to write out
356/// @param compression whether and how to compress the data (either COMPRESS_NONE,
357/// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
358/// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
359/// without Blosc support.
360/// @details This default implementation is instantiated only for types
361/// whose size can be determined by the sizeof() operator.
362template<typename T>
363inline void
364writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
365{
366 if (compression & COMPRESS_BLOSC) {
367 bloscToStream(os, reinterpret_cast<const char*>(data), sizeof(T), count);
368 } else if (compression & COMPRESS_ZIP) {
369 zipToStream(os, reinterpret_cast<const char*>(data), sizeof(T) * count);
370 } else {
371 os.write(reinterpret_cast<const char*>(data), sizeof(T) * count);
372 }
373}
374
375/// Specialization for std::string output
376template<>
377 inline void
378 writeData<std::string>(std::ostream& os, const std::string* data, Index count,
379 uint32_t /*compression*/) ///< @todo add compression
380{
381 for (Index i = 0; i < count; ++i) {
382 const size_t len = data[i].size();
383 os << len;
384 os.write(data[i].c_str(), len+1);
385 //os.write(&(data[i][0]), len );
386 }
387}
388
389/// HalfWriter wraps a static function, write(), that is analogous to writeData(), above,
390/// except that it is partially specialized for floating-point types in order to quantize
391/// floating-point values to 16-bit half float. A wrapper class is required because
392/// only classes, not functions, can be partially specialized.
393 template<bool IsReal, typename T> struct HalfWriter;
394/// Partial specialization for non-floating-point types (no float to half quantization)
395template<typename T>
396 struct HalfWriter</*IsReal=*/false, T> {
397 static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
398 return writeDataSize(data, count, compression);
399 }
400 static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
401 writeData(os, data, count, compression);
402 }
403};
404/// Partial specialization for floating-point types
405template<typename T>
406 struct HalfWriter</*IsReal=*/true, T> {
407 using HalfT = typename RealToHalf<T>::HalfT;
408 static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
409 if (count < 1) return size_t(0);
410 // Convert full float values to half float, then output the half float array.
411 std::vector<HalfT> halfData(count);
412 for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
413 return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
414 }
415 static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
416 if (count < 1) return;
417 // Convert full float values to half float, then output the half float array.
418 std::vector<HalfT> halfData(count);
419 for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
420 writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
421 }
422};
423#ifdef _WIN32
424/// Specialization to avoid double to float warnings in MSVC
425template<>
426struct HalfWriter</*IsReal=*/true, double> {
427 using HalfT = RealToHalf<double>::HalfT;
428 static inline size_t writeSize(const double* data, Index count, uint32_t compression)
429 {
430 if (count < 1) return size_t(0);
431 // Convert full float values to half float, then output the half float array.
432 std::vector<HalfT> halfData(count);
433 for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
434 return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
435 }
436 static inline void write(std::ostream& os, const double* data, Index count,
437 uint32_t compression)
438 {
439 if (count < 1) return;
440 // Convert full float values to half float, then output the half float array.
441 std::vector<HalfT> halfData(count);
442 for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
443 writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
444 }
445};
446#endif // _WIN32
447
448
449////////////////////////////////////////
450
451
452/// Populate the given buffer with @a destCount values of type @c ValueT
453/// read from the given stream, taking into account that the stream might
454/// have been compressed via one of several supported schemes.
455/// [Mainly for internal use]
456/// @param is a stream from which to read data (possibly compressed,
457/// depending on the stream's compression settings)
458/// @param destBuf a buffer into which to read values of type @c ValueT
459/// @param destCount the number of values to be stored in the buffer
460/// @param valueMask a bitmask (typically, a node's value mask) indicating
461/// which positions in the buffer correspond to active values
462/// @param fromHalf if true, read 16-bit half floats from the input stream
463/// and convert them to full floats
464template<typename ValueT, typename MaskT>
465inline void
466readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
467 const MaskT& valueMask, bool fromHalf)
468{
469 // Get the stream's compression settings.
470 auto meta = getStreamMetadataPtr(is);
471 const uint32_t compression = getDataCompression(is);
472 const bool maskCompressed = compression & COMPRESS_ACTIVE_MASK;
473
474 const bool seek = (destBuf == nullptr);
475 OPENVDB_ASSERT(!seek || (!meta || meta->seekable()));
476
477 // Get delayed load metadata if it exists
478
479 DelayedLoadMetadata::Ptr delayLoadMeta;
480 uint64_t leafIndex(0);
481 if (seek && meta && meta->delayedLoadMeta()) {
482 delayLoadMeta =
483 meta->gridMetadata().getMetadata<DelayedLoadMetadata>("file_delayed_load");
484 leafIndex = meta->leaf();
485 }
486
487 int8_t metadata = NO_MASK_AND_ALL_VALS;
488 if (getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) {
489 // Read the flag that specifies what, if any, additional metadata
490 // (selection mask and/or inactive value(s)) is saved.
491 if (seek && !maskCompressed) {
492 is.seekg(/*bytes=*/1, std::ios_base::cur);
493 } else if (seek && delayLoadMeta) {
494 metadata = delayLoadMeta->getMask(leafIndex);
495 is.seekg(/*bytes=*/1, std::ios_base::cur);
496 } else {
497 is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
498 }
499 }
500
501 ValueT background = zeroVal<ValueT>();
502 if (const void* bgPtr = getGridBackgroundValuePtr(is)) {
503 background = *static_cast<const ValueT*>(bgPtr);
504 }
505 ValueT inactiveVal1 = background;
506 ValueT inactiveVal0 =
507 ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background));
508
509 if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
510 metadata == MASK_AND_ONE_INACTIVE_VAL ||
511 metadata == MASK_AND_TWO_INACTIVE_VALS)
512 {
513 // Read one of at most two distinct inactive values.
514 if (seek) {
515 is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
516 } else {
517 is.read(reinterpret_cast<char*>(&inactiveVal0), /*bytes=*/sizeof(ValueT));
518 }
519 if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
520 // Read the second of two distinct inactive values.
521 if (seek) {
522 is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
523 } else {
524 is.read(reinterpret_cast<char*>(&inactiveVal1), /*bytes=*/sizeof(ValueT));
525 }
526 }
527 }
528
529 MaskT selectionMask;
530 if (metadata == MASK_AND_NO_INACTIVE_VALS ||
531 metadata == MASK_AND_ONE_INACTIVE_VAL ||
532 metadata == MASK_AND_TWO_INACTIVE_VALS)
533 {
534 // For use in mask compression (only), read the bitmask that selects
535 // between two distinct inactive values.
536 if (seek) {
537 is.seekg(/*bytes=*/selectionMask.memUsage(), std::ios_base::cur);
538 } else {
539 selectionMask.load(is);
540 }
541 }
542
543 ValueT* tempBuf = destBuf;
544 std::unique_ptr<ValueT[]> scopedTempBuf;
545
546 Index tempCount = destCount;
547
548 if (maskCompressed && metadata != NO_MASK_AND_ALL_VALS
549 && getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION)
550 {
551 tempCount = valueMask.countOn();
552 if (!seek && tempCount != destCount) {
553 // If this node has inactive voxels, allocate a temporary buffer
554 // into which to read just the active values.
555 scopedTempBuf.reset(new ValueT[tempCount]);
556 tempBuf = scopedTempBuf.get();
557 }
558 }
559
560 // Read in the buffer.
561 if (fromHalf) {
562 HalfReader<RealToHalf<ValueT>::isReal, ValueT>::read(
563 is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
564 } else {
565 readData<ValueT>(
566 is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
567 }
568
569 // If mask compression is enabled and the number of active values read into
570 // the temp buffer is smaller than the size of the destination buffer,
571 // then there are missing (inactive) values.
572 if (!seek && maskCompressed && tempCount != destCount) {
573 // Restore inactive values, using the background value and, if available,
574 // the inside/outside mask. (For fog volumes, the destination buffer is assumed
575 // to be initialized to background value zero, so inactive values can be ignored.)
576 for (Index destIdx = 0, tempIdx = 0; destIdx < MaskT::SIZE; ++destIdx) {
577 if (valueMask.isOn(destIdx)) {
578 // Copy a saved active value into this node's buffer.
579 destBuf[destIdx] = tempBuf[tempIdx];
580 ++tempIdx;
581 } else {
582 // Reconstruct an unsaved inactive value and copy it into this node's buffer.
583 destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
584 }
585 }
586 }
587}
588
589
590template<typename ValueT, typename MaskT>
591inline size_t
592writeCompressedValuesSize(ValueT* srcBuf, Index srcCount,
593 const MaskT& valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
594{
595 using NonConstValueT = typename std::remove_const<ValueT>::type;
596
597 const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
598
599 Index tempCount = srcCount;
600 ValueT* tempBuf = srcBuf;
601 std::unique_ptr<NonConstValueT[]> scopedTempBuf;
602
603 if (maskCompress && maskMetadata != NO_MASK_AND_ALL_VALS) {
604
605 tempCount = 0;
606
607 Index64 onVoxels = valueMask.countOn();
608 if (onVoxels > Index64(0)) {
609 // Create a new array to hold just the active values.
610 scopedTempBuf.reset(new NonConstValueT[onVoxels]);
611 NonConstValueT* localTempBuf = scopedTempBuf.get();
612
613 // Copy active values to a new, contiguous array.
614 for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
615 localTempBuf[tempCount] = srcBuf[it.pos()];
616 }
617
618 tempBuf = scopedTempBuf.get();
619 }
620 }
621
622 // Return the buffer size.
623 if (toHalf) {
624 return HalfWriter<RealToHalf<NonConstValueT>::isReal, NonConstValueT>::writeSize(
625 tempBuf, tempCount, compress);
626 } else {
627 return writeDataSize<NonConstValueT>(tempBuf, tempCount, compress);
628 }
629}
630
631
632/// Write @a srcCount values of type @c ValueT to the given stream, optionally
633/// after compressing the values via one of several supported schemes.
634/// [Mainly for internal use]
635/// @param os a stream to which to write data (possibly compressed, depending
636/// on the stream's compression settings)
637/// @param srcBuf a buffer containing values of type @c ValueT to be written
638/// @param srcCount the number of values stored in the buffer
639/// @param valueMask a bitmask (typically, a node's value mask) indicating
640/// which positions in the buffer correspond to active values
641/// @param childMask a bitmask (typically, a node's child mask) indicating
642/// which positions in the buffer correspond to child node pointers
643/// @param toHalf if true, convert floating-point values to 16-bit half floats
644template<typename ValueT, typename MaskT>
645inline void
646writeCompressedValues(std::ostream& os, ValueT* srcBuf, Index srcCount,
647 const MaskT& valueMask, const MaskT& childMask, bool toHalf)
648{
649 // Get the stream's compression settings.
650 const uint32_t compress = getDataCompression(os);
651 const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
652
653 Index tempCount = srcCount;
654 ValueT* tempBuf = srcBuf;
655 std::unique_ptr<ValueT[]> scopedTempBuf;
656
657 int8_t metadata = NO_MASK_AND_ALL_VALS;
658
659 if (!maskCompress) {
660 os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
661 } else {
662 // A valid level set's inactive values are either +background (outside)
663 // or -background (inside), and a fog volume's inactive values are all zero.
664 // Rather than write out all of these values, we can store just the active values
665 // (given that the value mask specifies their positions) and, if necessary,
666 // an inside/outside bitmask.
667
668 const ValueT zero = zeroVal<ValueT>();
669 ValueT background = zero;
670 if (const void* bgPtr = getGridBackgroundValuePtr(os)) {
671 background = *static_cast<const ValueT*>(bgPtr);
672 }
673
674 MaskCompress<ValueT, MaskT> maskCompressData(valueMask, childMask, srcBuf, background);
675 metadata = maskCompressData.metadata;
676
677 os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
678
679 if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
680 metadata == MASK_AND_ONE_INACTIVE_VAL ||
681 metadata == MASK_AND_TWO_INACTIVE_VALS)
682 {
683 if (!toHalf) {
684 // Write one of at most two distinct inactive values.
685 os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[0]), sizeof(ValueT));
686 if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
687 // Write the second of two distinct inactive values.
688 os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[1]), sizeof(ValueT));
689 }
690 } else {
691 // Write one of at most two distinct inactive values.
692 ValueT truncatedVal = static_cast<ValueT>(truncateRealToHalf(maskCompressData.inactiveVal[0]));
693 os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
694 if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
695 // Write the second of two distinct inactive values.
696 truncatedVal = truncateRealToHalf(maskCompressData.inactiveVal[1]);
697 os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
698 }
699 }
700 }
701
702 if (metadata == NO_MASK_AND_ALL_VALS) {
703 // If there are more than two unique inactive values, the entire input buffer
704 // needs to be saved (both active and inactive values).
705 /// @todo Save the selection mask as long as most of the inactive values
706 /// are one of two values?
707 } else {
708 // Create a new array to hold just the active values.
709 scopedTempBuf.reset(new ValueT[srcCount]);
710 tempBuf = scopedTempBuf.get();
711
712 if (metadata == NO_MASK_OR_INACTIVE_VALS ||
713 metadata == NO_MASK_AND_MINUS_BG ||
714 metadata == NO_MASK_AND_ONE_INACTIVE_VAL)
715 {
716 // Copy active values to the contiguous array.
717 tempCount = 0;
718 for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
719 tempBuf[tempCount] = srcBuf[it.pos()];
720 }
721 } else {
722 // Copy active values to a new, contiguous array and populate a bitmask
723 // that selects between two distinct inactive values.
724 MaskT selectionMask;
725 tempCount = 0;
726 for (Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
727 if (valueMask.isOn(srcIdx)) { // active value
728 tempBuf[tempCount] = srcBuf[srcIdx];
729 ++tempCount;
730 } else { // inactive value
731 if (MaskCompress<ValueT, MaskT>::eq(srcBuf[srcIdx], maskCompressData.inactiveVal[1])) {
732 selectionMask.setOn(srcIdx); // inactive value 1
733 } // else inactive value 0
734 }
735 }
736 OPENVDB_ASSERT(tempCount == valueMask.countOn());
737
738 // Write out the mask that selects between two inactive values.
739 selectionMask.save(os);
740 }
741 }
742 }
743
744 // Write out the buffer.
745 if (toHalf) {
746 HalfWriter<RealToHalf<ValueT>::isReal, ValueT>::write(os, tempBuf, tempCount, compress);
747 } else {
748 writeData(os, tempBuf, tempCount, compress);
749 }
750}
751
752} // namespace io
753} // namespace OPENVDB_VERSION_NAME
754} // namespace openvdb
755
756#endif // OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
Assert.h
OPENVDB_ASSERT
#define OPENVDB_ASSERT(X)
Definition Assert.h:41
DelayedLoadMetadata.h
Math.h
General-purpose arithmetic and comparison routines, most of which accept arbitrary value types (or at...
MetaMap.h
OPENVDB_API
#define OPENVDB_API
Definition Platform.h:291
openvdb::v12_1::io::DelayedLoadMetadata
Store a buffer of data that can be optionally used during reading for faster delayed-load I/O perform...
Definition DelayedLoadMetadata.h:22
openvdb::v12_1::io::DelayedLoadMetadata::Ptr
SharedPtr< DelayedLoadMetadata > Ptr
Definition DelayedLoadMetadata.h:24
io.h
openvdb::v12_1::compression
Definition StreamCompression.h:34
openvdb::v12_1::io::getDataCompression
OPENVDB_API uint32_t getDataCompression(std::ios_base &)
Return a bitwise OR of compression option flags (COMPRESS_ZIP, COMPRESS_ACTIVE_MASK,...
openvdb::v12_1::io::zipToStreamSize
OPENVDB_API size_t zipToStreamSize(const char *data, size_t numBytes)
openvdb::v12_1::io::writeCompressedValues
void writeCompressedValues(std::ostream &os, ValueT *srcBuf, Index srcCount, const MaskT &valueMask, const MaskT &childMask, bool toHalf)
Definition Compression.h:646
openvdb::v12_1::io::getFormatVersion
OPENVDB_API uint32_t getFormatVersion(std::ios_base &)
Return the file format version number associated with the given input stream.
openvdb::v12_1::io::zipToStream
OPENVDB_API void zipToStream(std::ostream &, const char *data, size_t numBytes)
openvdb::v12_1::io::COMPRESS_BLOSC
@ COMPRESS_BLOSC
Definition Compression.h:57
openvdb::v12_1::io::COMPRESS_NONE
@ COMPRESS_NONE
Definition Compression.h:54
openvdb::v12_1::io::COMPRESS_ACTIVE_MASK
@ COMPRESS_ACTIVE_MASK
Definition Compression.h:56
openvdb::v12_1::io::COMPRESS_ZIP
@ COMPRESS_ZIP
Definition Compression.h:55
openvdb::v12_1::io::truncateRealToHalf
T truncateRealToHalf(const T &val)
Return the given value truncated to 16-bit float precision.
Definition Compression.h:217
openvdb::v12_1::io::unzipFromStream
OPENVDB_API void unzipFromStream(std::istream &, char *data, size_t numBytes)
openvdb::v12_1::io::bloscToStreamSize
OPENVDB_API size_t bloscToStreamSize(const char *data, size_t valSize, size_t numVals)
openvdb::v12_1::io::bloscToStream
OPENVDB_API void bloscToStream(std::ostream &, const char *data, size_t valSize, size_t numVals)
openvdb::v12_1::io::readCompressedValues
void readCompressedValues(std::istream &is, ValueT *destBuf, Index destCount, const MaskT &valueMask, bool fromHalf)
Definition Compression.h:466
openvdb::v12_1::io::writeData
void writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition Compression.h:364
openvdb::v12_1::io::getGridBackgroundValuePtr
OPENVDB_API const void * getGridBackgroundValuePtr(std::ios_base &)
Return a pointer to the background value of the grid currently being read from or written to the give...
openvdb::v12_1::io::compressionToString
OPENVDB_API std::string compressionToString(uint32_t flags)
Return a string describing the given compression flags.
openvdb::v12_1::io::getStreamMetadataPtr
OPENVDB_API SharedPtr< StreamMetadata > getStreamMetadataPtr(std::ios_base &)
Return a shared pointer to an object that stores metadata (file format, compression scheme,...
openvdb::v12_1::io::bloscFromStream
OPENVDB_API void bloscFromStream(std::istream &, char *data, size_t numBytes)
openvdb::v12_1::io::MASK_AND_NO_INACTIVE_VALS
@ MASK_AND_NO_INACTIVE_VALS
Definition Compression.h:73
openvdb::v12_1::io::MASK_AND_TWO_INACTIVE_VALS
@ MASK_AND_TWO_INACTIVE_VALS
Definition Compression.h:75
openvdb::v12_1::io::NO_MASK_AND_MINUS_BG
@ NO_MASK_AND_MINUS_BG
Definition Compression.h:71
openvdb::v12_1::io::NO_MASK_AND_ONE_INACTIVE_VAL
@ NO_MASK_AND_ONE_INACTIVE_VAL
Definition Compression.h:72
openvdb::v12_1::io::MASK_AND_ONE_INACTIVE_VAL
@ MASK_AND_ONE_INACTIVE_VAL
Definition Compression.h:74
openvdb::v12_1::io::NO_MASK_AND_ALL_VALS
@ NO_MASK_AND_ALL_VALS
Definition Compression.h:76
openvdb::v12_1::io::NO_MASK_OR_INACTIVE_VALS
@ NO_MASK_OR_INACTIVE_VALS
Definition Compression.h:70
openvdb::v12_1::io::readData
void readData(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Read data from a stream.
Definition Compression.h:248
openvdb::v12_1::io::writeCompressedValuesSize
size_t writeCompressedValuesSize(ValueT *srcBuf, Index srcCount, const MaskT &valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
Definition Compression.h:592
openvdb::v12_1::io::writeDataSize
size_t writeDataSize(const T *data, Index count, uint32_t compression)
Definition Compression.h:325
openvdb::v12_1::math::Vec2d
Vec2< double > Vec2d
Definition Vec2.h:533
openvdb::v12_1::math::Vec2s
Vec2< float > Vec2s
Definition Vec2.h:532
openvdb::v12_1::math::Vec3d
Vec3< double > Vec3d
Definition Vec3.h:665
openvdb::v12_1::math::negative
T negative(const T &val)
Return the unary negation of the given value.
Definition Math.h:128
openvdb::v12_1::math::isExactlyEqual
bool isExactlyEqual(const T0 &a, const T1 &b)
Return true if a is exactly equal to b.
Definition Math.h:443
openvdb::v12_1::math::half
internal::half half
Definition Types.h:29
openvdb::v12_1::math::Vec3s
Vec3< float > Vec3s
Definition Vec3.h:664
openvdb::v12_1::Vec2H
math::Vec2< math::half > Vec2H
Definition Types.h:66
openvdb::v12_1::Index
Index32 Index
Definition Types.h:54
openvdb::v12_1::Vec3H
math::Vec3< math::half > Vec3H
Definition Types.h:75
openvdb::v12_1::zeroVal
constexpr T zeroVal()
Return the value of type T that corresponds to zero.
Definition Math.h:70
openvdb::v12_1::OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION
@ OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION
Definition version.h.in:262
openvdb::v12_1::Index32
uint32_t Index32
Definition Types.h:52
openvdb::v12_1::Index64
uint64_t Index64
Definition Types.h:53
openvdb
Definition Exceptions.h:13
Types.h
openvdb::v12_1::io::HalfReader< false, T >::read
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition Compression.h:297
openvdb::v12_1::io::HalfReader< true, T >::HalfT
typename RealToHalf< T >::HalfT HalfT
Definition Compression.h:305
openvdb::v12_1::io::HalfReader< true, T >::read
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition Compression.h:306
openvdb::v12_1::io::HalfReader
Definition Compression.h:293
openvdb::v12_1::io::HalfWriter< false, T >::writeSize
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition Compression.h:397
openvdb::v12_1::io::HalfWriter< false, T >::write
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition Compression.h:400
openvdb::v12_1::io::HalfWriter< true, T >::writeSize
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition Compression.h:408
openvdb::v12_1::io::HalfWriter< true, T >::HalfT
typename RealToHalf< T >::HalfT HalfT
Definition Compression.h:407
openvdb::v12_1::io::HalfWriter< true, T >::write
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition Compression.h:415
openvdb::v12_1::io::HalfWriter
Definition Compression.h:393
openvdb::v12_1::io::MaskCompress
Definition Compression.h:82
openvdb::v12_1::io::MaskCompress::metadata
int8_t metadata
Definition Compression.h:163
openvdb::v12_1::io::MaskCompress::MaskCompress
MaskCompress(const MaskT &valueMask, const MaskT &childMask, const ValueT *srcBuf, const ValueT &background)
Definition Compression.h:88
openvdb::v12_1::io::MaskCompress::eq
static bool eq(const ValueT &a, const ValueT &b)
Definition Compression.h:84
openvdb::v12_1::io::MaskCompress::inactiveVal
ValueT inactiveVal[2]
Definition Compression.h:164
openvdb::v12_1::io::RealToHalf< Vec2d >::HalfT
Vec2H HalfT
Definition Compression.h:197
openvdb::v12_1::io::RealToHalf< Vec2d >::isReal
@ isReal
Definition Compression.h:196
openvdb::v12_1::io::RealToHalf< Vec2d >::convert
static HalfT convert(const Vec2d &val)
Definition Compression.h:199
openvdb::v12_1::io::RealToHalf< Vec2s >::HalfT
Vec2H HalfT
Definition Compression.h:192
openvdb::v12_1::io::RealToHalf< Vec2s >::isReal
@ isReal
Definition Compression.h:191
openvdb::v12_1::io::RealToHalf< Vec2s >::convert
static HalfT convert(const Vec2s &val)
Definition Compression.h:193
openvdb::v12_1::io::RealToHalf< Vec3d >::isReal
@ isReal
Definition Compression.h:207
openvdb::v12_1::io::RealToHalf< Vec3d >::HalfT
Vec3H HalfT
Definition Compression.h:208
openvdb::v12_1::io::RealToHalf< Vec3d >::convert
static HalfT convert(const Vec3d &val)
Definition Compression.h:210
openvdb::v12_1::io::RealToHalf< Vec3s >::HalfT
Vec3H HalfT
Definition Compression.h:203
openvdb::v12_1::io::RealToHalf< Vec3s >::isReal
@ isReal
Definition Compression.h:202
openvdb::v12_1::io::RealToHalf< Vec3s >::convert
static HalfT convert(const Vec3s &val)
Definition Compression.h:204
openvdb::v12_1::io::RealToHalf< double >::HalfT
math::half HalfT
Definition Compression.h:186
openvdb::v12_1::io::RealToHalf< double >::isReal
@ isReal
Definition Compression.h:185
openvdb::v12_1::io::RealToHalf< double >::convert
static HalfT convert(double val)
Definition Compression.h:188
openvdb::v12_1::io::RealToHalf< float >::convert
static HalfT convert(float val)
Definition Compression.h:182
openvdb::v12_1::io::RealToHalf< float >::isReal
@ isReal
Definition Compression.h:180
openvdb::v12_1::io::RealToHalf< float >::HalfT
math::half HalfT
Definition Compression.h:181
openvdb::v12_1::io::RealToHalf
RealToHalf and its specializations define a mapping from floating-point data types to analogous half ...
Definition Compression.h:174
openvdb::v12_1::io::RealToHalf::isReal
@ isReal
Definition Compression.h:175
openvdb::v12_1::io::RealToHalf::HalfT
T HalfT
Definition Compression.h:176
openvdb::v12_1::io::RealToHalf::convert
static HalfT convert(const T &val)
Definition Compression.h:177
OPENVDB_VERSION_NAME
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition version.h.in:121
OPENVDB_USE_VERSION_NAMESPACE
#define OPENVDB_USE_VERSION_NAMESPACE
Definition version.h.in:218