40#include <libdap/Array.h>
41#include <libdap/util.h>
42#include <libdap/D4Attributes.h>
45#include <BESNotFoundError.h>
46#include <BESInternalError.h>
49#include "DmrppTypeFactory.h"
50#include "DmrppD4Group.h"
53#define H5S_MAX_RANK 32
54#define H5O_LAYOUT_NDIMS (H5S_MAX_RANK+1)
67typedef struct H5D_chunk_rec_t {
68 hsize_t scaled[H5O_LAYOUT_NDIMS];
79namespace build_dmrpp_util {
83#define VERBOSE(x) do { if (verbose) (x); } while(false)
106string h5_filter_name(
int type) {
109 case H5Z_FILTER_NONE:
110 name =
"H5Z_FILTER_NONE";
112 case H5Z_FILTER_DEFLATE:
113 name =
"H5Z_FILTER_DEFLATE";
115 case H5Z_FILTER_SHUFFLE:
116 name =
"H5Z_FILTER_SHUFFLE";
118 case H5Z_FILTER_FLETCHER32:
119 name =
"H5Z_FILTER_FLETCHER32";
121 case H5Z_FILTER_SZIP:
122 name =
"H5Z_FILTER_SZIP";
124 case H5Z_FILTER_NBIT:
125 name =
"H5Z_FILTER_NBIT";
127 case H5Z_FILTER_SCALEOFFSET:
128 name =
"H5Z_FILTER_SCALEOFFSET";
132 ostringstream oss(
"ERROR! Unknown HDF5 FILTER! type: ", std::ios::ate);
147static void set_filter_information(hid_t dataset_id,
DmrppCommon *dc) {
148 hid_t plist_id = H5Dget_create_plist(dataset_id);
151 int numfilt = H5Pget_nfilters(plist_id);
152 VERBOSE(cerr <<
"Number of filters associated with dataset: " << numfilt << endl);
155 for (
int filter = 0; filter < numfilt; filter++) {
157 unsigned int flags, filter_info;
158 H5Z_filter_t filter_type = H5Pget_filter2(plist_id, filter, &flags, &nelmts,
159 nullptr, 0,
nullptr, &filter_info);
160 VERBOSE(cerr <<
"Found H5 Filter Type: " << h5_filter_name(filter_type) <<
" (" << filter_type <<
")" << endl);
161 switch (filter_type) {
162 case H5Z_FILTER_DEFLATE:
163 filters.append(
"deflate ");
165 case H5Z_FILTER_SHUFFLE:
166 filters.append(
"shuffle ");
168 case H5Z_FILTER_FLETCHER32:
169 filters.append(
"fletcher32 ");
172 ostringstream oss(
"Unsupported HDF5 filter: ", std::ios::ate);
174 oss <<
" (" << h5_filter_name(filter_type) <<
")";
179 filters = filters.substr(0, filters.length() - 1);
191is_hdf5_fill_value_defined(hid_t dataset_id)
196 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
199 if ( (plist_id = H5Dget_create_plist(dataset_id)) < 0 )
200 throw BESInternalError(
"Unable to open HDF5 dataset id.", __FILE__, __LINE__);
203 H5D_fill_value_t status;
204 if ( (H5Pfill_value_defined(plist_id, &status)) < 0 ) {
206 throw BESInternalError(
"Unable to access HDF5 Fillvalue information.", __FILE__, __LINE__);
211 return status != H5D_FILL_VALUE_UNDEFINED;
225get_value_as_string(hid_t h5_type_id, vector<char> &value)
227 H5T_class_t class_type = H5Tget_class(h5_type_id);
229 switch (class_type) {
231 sign = H5Tget_sign(h5_type_id);
232 switch (H5Tget_size(h5_type_id)) {
234 if (sign == H5T_SGN_2)
235 return to_string(*(int8_t *)(value.data()));
237 return to_string(*(uint8_t *)(value.data()));
240 if (sign == H5T_SGN_2)
241 return to_string(*(int16_t *)(value.data()));
243 return to_string(*(uint16_t *)(value.data()));
246 if (sign == H5T_SGN_2)
247 return to_string(*(int32_t *)(value.data()));
249 return to_string(*(uint32_t *)(value.data()));
252 if (sign == H5T_SGN_2)
253 return to_string(*(int64_t *)(value.data()));
255 return to_string(*(uint64_t *)(value.data()));
258 throw BESInternalError(
"Unable extract integer fill value.", __FILE__, __LINE__);
264 switch (H5Tget_size(h5_type_id)) {
266 oss << *(
float *) (value.data());
270 oss << *(
double *) (value.data());
274 throw BESInternalError(
"Unable extract float fill value.", __FILE__, __LINE__);
280 return "unsupported-string";
282 return "unsupported-array";
284 return "unsupported-compound";
297string get_hdf5_fill_value(hid_t dataset_id)
300 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
303 hid_t plist_id = H5Dget_create_plist(dataset_id);
305 throw BESInternalError(
"Unable to open HDF5 dataset id.", __FILE__, __LINE__);
308 hid_t dtype_id = H5Dget_type(dataset_id);
310 throw BESInternalError(
"Unable to get HDF5 dataset type id.", __FILE__, __LINE__);
312 vector<char> value(H5Tget_size(dtype_id));
313 if (H5Pget_fill_value(plist_id, dtype_id, value.data()) < 0)
314 throw BESInternalError(
"Unable to access HDF5 Fill Value.", __FILE__, __LINE__);
318 return get_value_as_string(dtype_id, value);
336static void get_variable_chunk_info(hid_t dataset,
DmrppCommon *dc) {
337 std::string byteOrder;
338 H5T_order_t byte_order;
341 bool fill_value_defined = is_hdf5_fill_value_defined(dataset);
342 if (fill_value_defined) {
343 string fill_value = get_hdf5_fill_value(dataset);
349 hid_t dcpl = H5Dget_create_plist(dataset);
350 uint8_t layout_type = H5Pget_layout(dcpl);
352 hid_t fspace_id = H5Dget_space(dataset);
353 hid_t dtypeid = H5Dget_type(dataset);
355 byte_order = H5Tget_order(dtypeid);
356 switch (byte_order) {
367 ostringstream oss(
"Unsupported HDF5 dataset byteOrder: ", std::ios::ate);
368 oss << byte_order <<
".";
372 int dataset_rank = H5Sget_simple_extent_ndims(fspace_id);
374 size_t dsize = H5Tget_size(dtypeid);
377 switch (layout_type) {
379 case H5D_CONTIGUOUS: {
380 VERBOSE(cerr <<
"Storage: contiguous" << endl);
382 haddr_t cont_addr = H5Dget_offset(dataset);
383 hsize_t cont_size = H5Dget_storage_size(dataset);
385 VERBOSE(cerr <<
" Addr: " << cont_addr << endl);
386 VERBOSE(cerr <<
" Size: " << cont_size << endl);
387 VERBOSE(cerr <<
"byteOrder: " << byteOrder << endl);
389 if (cont_size > 0 && dc) {
390 dc->
add_chunk(byteOrder, cont_size, cont_addr,
"" );
395 hsize_t num_chunks = 0;
396 herr_t status = H5Dget_num_chunks(dataset, fspace_id, &num_chunks);
398 throw BESInternalError(
"Could not get the number of chunks", __FILE__, __LINE__);
401 VERBOSE(cerr <<
"Storage: chunked." << endl);
402 VERBOSE(cerr <<
"Number of chunks is: " << num_chunks << endl);
405 set_filter_information(dataset, dc);
408 vector<hsize_t> chunk_dims(dataset_rank);
409 unsigned int chunk_rank = H5Pget_chunk(dcpl, dataset_rank, chunk_dims.data());
410 if (chunk_rank != dataset_rank)
412 "Found a chunk with rank different than the dataset's (aka variables') rank", __FILE__,
417 for (
unsigned int i = 0; i < num_chunks; ++i) {
418 vector<hsize_t> chunk_coords(dataset_rank);
422 status = H5Dget_chunk_info(dataset, fspace_id, i, chunk_coords.data(),
423 nullptr, &addr, &size);
425 VERBOSE(cerr <<
"ERROR" << endl);
426 throw BESInternalError(
"Cannot get HDF5 dataset storage info.", __FILE__, __LINE__);
429 VERBOSE(cerr <<
"chk_idk: " << i <<
", addr: " << addr <<
", size: " << size << endl);
430 if (dc) dc->
add_chunk(byteOrder, size, addr, chunk_coords);
437 VERBOSE(cerr <<
"Storage: compact" << endl);
439 size_t comp_size = H5Dget_storage_size(dataset);
440 VERBOSE(cerr <<
" Size: " << comp_size << endl);
442 if (comp_size == 0) {
443 throw BESInternalError(
"Cannot obtain the compact storage size.", __FILE__, __LINE__);
446 vector<uint8_t> values;
448 auto btp =
dynamic_cast<Array *
>(dc);
449 if (btp !=
nullptr) {
451 size_t memRequired = btp->length() * dsize;
453 if (comp_size != memRequired) {
454 throw BESInternalError(
"Compact storage size does not match D4Array.", __FILE__, __LINE__);
457 switch (btp->var()->type()) {
469 case dods_uint64_c: {
470 values.resize(memRequired);
471 get_data(dataset,
reinterpret_cast<void *
>(values.data()));
472 btp->set_read_p(
true);
473 btp->val2buf(
reinterpret_cast<void *
>(values.data()));
479 if (H5Tis_variable_str(dtypeid) > 0) {
480 vector<string> finstrval = {
""};
481 read_vlen_string(dataset, 1,
nullptr,
nullptr,
nullptr, finstrval);
482 btp->set_value(finstrval, (
int)finstrval.size());
483 btp->set_read_p(
true);
494 assert(btp->length() == 1);
495 values.resize(memRequired);
496 get_data(dataset,
reinterpret_cast<void *
>(values.data()));
497 string str(values.begin(), values.end());
498 vector<string> strings = {str};
499 btp->set_value(strings, (
int)strings.size());
500 btp->set_read_p(
true);
506 throw BESInternalError(
"Unsupported compact storage variable type.", __FILE__, __LINE__);
518 ostringstream oss(
"Unsupported HDF5 dataset layout type: ", std::ios::ate);
519 oss << layout_type <<
".";
538void get_chunks_for_all_variables(hid_t file, D4Group *group) {
540 for (
auto v = group->var_begin(), ve = group->var_end(); v != ve; ++v) {
543 D4Attributes *d4_attrs = (*v)->attributes();
545 throw BESInternalError(
"Expected to find an attribute table for " + (*v)->name() +
" but did not.",
551 const D4Attribute *attr = d4_attrs->get(
"fullnamepath");
562 if (attr->num_values() == 1)
563 FQN = attr->value(0);
566 BESDEBUG(
"dmrpp",
"Working on: " << FQN << endl);
567 dataset = H5Dopen2(file, FQN.c_str(), H5P_DEFAULT);
569 throw BESInternalError(
"HDF5 dataset '" + FQN +
"' cannot be opened.", __FILE__, __LINE__);
579 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
580 string FQN = (*v)->FQN();
581 BESDEBUG(
"dmrpp",
"Working on: " << FQN << endl);
582 dataset = H5Dopen2(file, FQN.c_str(), H5P_DEFAULT);
587 get_variable_chunk_info(dataset,
dynamic_cast<DmrppCommon *
>(*v));
591 for (
auto g = group->grp_begin(), ge = group->grp_end(); g != ge; ++g)
592 get_chunks_for_all_variables(file, *g);
600void add_chunk_information(
const string &h5_file_name,
DMRpp *dmrpp)
603 hid_t file = H5Fopen(h5_file_name.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
605 throw BESNotFoundError(
string(
"Error: HDF5 file '").append(h5_file_name).append(
"' cannot be opened."), __FILE__, __LINE__);
610 get_chunks_for_all_variables(file, dmrpp->root());
exception thrown if internal error encountered
error thrown if the resource requested cannot be found
Provide a way to print the DMR++ response.
Size and offset information of data included in DMR++ files.
virtual void set_fill_value_string(const std::string &fv)
Set the fill value (using a string)
virtual void set_uses_fill_value(bool ufv)
Set the uses_fill_value property.
void set_chunk_dimension_sizes(const std::vector< unsigned long long > &chunk_dims)
Set the value of the chunk dimension sizes given a vector of HDF5 hsize_t.
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal ve...
void set_filter(const std::string &value)
Set the value of the filters property.
void set_compact(bool value)
Set the value of the compact property.
void get_data(hid_t dset, void *buf)