multi_array_chunked_hdf5.hxx
|
 |
36 #ifndef VIGRA_MULTI_ARRAY_CHUNKED_HDF5_HXX
37 #define VIGRA_MULTI_ARRAY_CHUNKED_HDF5_HXX
41 #include "multi_array_chunked.hxx"
42 #include "hdf5impex.hxx"
45 #ifdef VIGRA_CHECK_BOUNDS
46 #define VIGRA_ASSERT_INSIDE(diff) \
47 vigra_precondition(this->isInside(diff), "Index out of bounds")
49 #define VIGRA_ASSERT_INSIDE(diff)
54 template <
unsigned int N,
class T,
class Alloc = std::allocator<T> >
55 class ChunkedArrayHDF5
56 :
public ChunkedArray<N, T>
61 :
public ChunkBase<N, T>
64 typedef typename MultiArrayShape<N>::type shape_type;
66 typedef value_type * pointer;
67 typedef value_type & reference;
69 Chunk(shape_type
const & shape, shape_type
const & start,
70 ChunkedArrayHDF5 * array, Alloc
const & alloc)
71 : ChunkBase<N, T>(detail::defaultStride(shape))
83 std::size_t size()
const
88 void write(
bool deallocate =
true)
90 if(this->pointer_ != 0)
92 if(!array_->file_.isReadOnly())
94 herr_t status = array_->file_.writeBlock(array_->dataset_, start_,
95 MultiArrayView<N, T>(shape_, this->strides_, this->pointer_));
96 vigra_postcondition(status >= 0,
97 "ChunkedArrayHDF5: write to dataset failed.");
101 alloc_.deallocate(this->pointer_, this->size());
109 if(this->pointer_ == 0)
111 this->pointer_ = alloc_.allocate(this->size());
112 herr_t status = array_->file_.readBlock(array_->dataset_, start_, shape_,
113 MultiArrayView<N, T>(shape_, this->strides_, this->pointer_));
114 vigra_postcondition(status >= 0,
115 "ChunkedArrayHDF5: read from dataset failed.");
117 return this->pointer_;
120 shape_type shape_, start_;
121 ChunkedArrayHDF5 * array_;
125 Chunk & operator=(Chunk
const &);
128 typedef ChunkedArray<N, T> base_type;
129 typedef MultiArray<N, SharedChunkHandle<N, T> > ChunkStorage;
131 typedef T value_type;
132 typedef value_type * pointer;
133 typedef value_type & reference;
135 ChunkedArrayHDF5(HDF5File
const & file, std::string
const & dataset,
137 shape_type
const & shape,
138 shape_type
const & chunk_shape=shape_type(),
139 ChunkedArrayOptions
const & options = ChunkedArrayOptions(),
140 Alloc
const & alloc = Alloc())
141 : ChunkedArray<N, T>(shape, chunk_shape, options),
143 dataset_name_(dataset),
145 compression_(options.compression_method),
151 ChunkedArrayHDF5(HDF5File
const & file, std::string
const & dataset,
153 ChunkedArrayOptions
const & options = ChunkedArrayOptions(),
154 Alloc
const & alloc = Alloc())
155 : ChunkedArray<N, T>(shape_type(), shape_type(), options),
157 dataset_name_(dataset),
159 compression_(options.compression_method),
167 bool exists = file_.existsDataset(dataset_name_);
169 if(mode == HDF5File::Replace)
171 mode = HDF5File::New;
173 else if(mode == HDF5File::Default)
176 mode = HDF5File::ReadOnly;
178 mode = HDF5File::New;
181 if(mode == HDF5File::ReadOnly)
184 vigra_precondition(!file_.isReadOnly(),
185 "ChunkedArrayHDF5(): 'mode' is incompatible with read-only file.");
187 vigra_precondition(exists || !file_.isReadOnly(),
188 "ChunkedArrayHDF5(): dataset does not exist, but file is read-only.");
190 if(!exists || mode == HDF5File::New)
209 if(compression_ == DEFAULT_COMPRESSION)
210 compression_ = ZLIB_FAST;
211 vigra_precondition(compression_ != LZ4,
212 "ChunkedArrayHDF5(): HDF5 does not support LZ4 compression.");
214 vigra_precondition(this->size() > 0,
215 "ChunkedArrayHDF5(): invalid shape.");
216 typename detail::HDF5TypeTraits<T>::value_type init(this->fill_scalar_);
217 dataset_ = file_.createDataset<N, T>(dataset_name_,
225 dataset_ = file_.getDatasetHandleShared(dataset_name_);
228 ArrayVector<hsize_t> fileShape(file_.getDatasetShape(dataset_name_));
229 typedef detail::HDF5TypeTraits<T> TypeTraits;
230 if(TypeTraits::numberOfBands() > 1)
232 vigra_precondition(fileShape.size() == N+1,
233 "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
234 vigra_precondition(fileShape[0] == TypeTraits::numberOfBands(),
235 "ChunkedArrayHDF5(file, dataset): dataset has wrong number of bands.");
236 shape_type shape(fileShape.begin()+1);
239 vigra_precondition(shape == this->shape_,
240 "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
244 this->shape_ = shape;
249 vigra_precondition(fileShape.size() == N,
250 "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
251 shape_type shape(fileShape.begin());
254 vigra_precondition(shape == this->shape_,
255 "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
259 this->shape_ = shape;
260 ChunkStorage(detail::computeChunkArrayShape(shape, this->bits_, this->mask_)).swap(this->handle_array_);
264 end = this->handle_array_.end();
267 i->chunk_state_.store(base_type::chunk_asleep);
282 void closeImpl(
bool force_destroy)
284 flushToDiskImpl(
true, force_destroy);
290 flushToDiskImpl(
false,
false);
293 void flushToDiskImpl(
bool destroy,
bool force_destroy)
295 if(file_.isReadOnly())
298 threading::lock_guard<threading::mutex> guard(*this->chunk_lock_);
300 end = this->handle_array_.end();
301 if(destroy && !force_destroy)
305 vigra_precondition(i->chunk_state_.load() <= 0,
306 "ChunkedArrayHDF5::close(): cannot close file because there are active chunks.");
308 i = this->handle_array_.begin();
312 Chunk * chunk =
static_cast<Chunk*
>(i->pointer_);
328 virtual bool isReadOnly()
const
330 return file_.isReadOnly();
333 virtual pointer loadChunk(ChunkBase<N, T> ** p, shape_type
const & index)
335 vigra_precondition(file_.isOpen(),
336 "ChunkedArrayHDF5::loadChunk(): file was already closed.");
339 *p =
new Chunk(this->chunkShape(index), index*this->chunk_shape_,
this, alloc_);
340 this->overhead_bytes_ +=
sizeof(Chunk);
342 return static_cast<Chunk *
>(*p)->read();
345 virtual bool unloadChunk(ChunkBase<N, T> * chunk,
bool )
349 static_cast<Chunk *
>(chunk)->write();
353 virtual std::string backend()
const
355 return "ChunkedArrayHDF5<'" + file_.filename() +
"/" + dataset_name_ +
"'>";
358 virtual std::size_t dataBytes(ChunkBase<N,T> * c)
const
360 return c->pointer_ == 0
362 :
static_cast<Chunk*
>(c)->size()*
sizeof(T);
365 virtual std::size_t overheadBytesPerChunk()
const
367 return sizeof(Chunk) +
sizeof(SharedChunkHandle<N, T>);
370 std::string fileName()
const
372 return file_.filename();
375 std::string datasetName()
const
377 return dataset_name_;
381 std::string dataset_name_;
382 HDF5HandleShared dataset_;
383 CompressionMethod compression_;
389 #undef VIGRA_ASSERT_INSIDE