31#include "config_hdf5.h"
37#include <libdap/InternalErr.h>
39#include <libdap/Str.h>
43#include "ObjMemCache.h"
49BaseType *HDF5CFArray::ptr_duplicate()
55bool HDF5CFArray::read()
58 BESDEBUG(
"h5",
"Coming to HDF5CFArray read "<<endl);
62 if((
nullptr == HDF5RequestHandler::get_lrdata_mem_cache()) &&
63 nullptr == HDF5RequestHandler::get_srdata_mem_cache()){
69 short use_cache_flag = 0;
72 if(HDF5RequestHandler::get_srdata_mem_cache() !=
nullptr) {
73 if(((cvtype == CV_EXIST) && (islatlon !=
true)) || (cvtype == CV_NONLATLON_MISS)
74 || (cvtype == CV_FILLINDEX) ||(cvtype == CV_MODIFY) ||(cvtype == CV_SPECIAL)){
76 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==
true)
82 if(use_cache_flag !=1) {
84 if(HDF5RequestHandler::get_lrdata_mem_cache() !=
nullptr) {
89 if(HDF5RequestHandler::get_common_cache_dirs() ==
false) {
90 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
91 || (cvtype == CV_EXIST && islatlon ==
true)) {
96 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==
true)
104 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
105 || (cvtype == CV_EXIST && islatlon ==
true)) {
107 vector<string> cur_lrd_non_cache_dir_list;
108 HDF5RequestHandler::get_lrd_non_cache_dir_list(cur_lrd_non_cache_dir_list);
111 if( (cur_lrd_non_cache_dir_list.empty()) ||
112 (
"" == check_str_sect_in_list(cur_lrd_non_cache_dir_list,filename,
'/'))) {
115 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==
true)
121 vector<string> cur_lrd_var_cache_file_list;
122 HDF5RequestHandler::get_lrd_var_cache_file_list(cur_lrd_var_cache_file_list);
123 if(cur_lrd_var_cache_file_list.empty() ==
false){
128 if(
true == check_var_cache_files(cur_lrd_var_cache_file_list,filename,varname)){
135 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==
true)
143 if(0 == use_cache_flag)
150 if( 3 == use_cache_flag){
151 vector<string> cur_cache_dlist;
152 HDF5RequestHandler::get_lrd_cache_dir_list(cur_cache_dlist);
153 string cache_dir = check_str_sect_in_list(cur_cache_dlist,filename,
'/');
155 cache_key = cache_dir + varname;
157 cache_key = filename + varname;
165 cache_key = filename + varname;
167 handle_data_with_mem_cache(dtype,total_elems,use_cache_flag,cache_key,is_dap4);
180 vector<hsize_t> hoffset;
181 vector<hsize_t>hcount;
182 vector<hsize_t>hstep;
186 throw InternalErr (__FILE__, __LINE__,
187 "The number of dimension of the variable is <=0 for an array.");
193 hoffset.resize(rank);
196 nelms = format_constraint (offset.data(), step.data(), count.data());
197 for (
int i = 0; i <rank; i++) {
198 hoffset[i] = (hsize_t) offset[i];
199 hcount[i] = (hsize_t) count[i];
200 hstep[i] = (hsize_t) step[i];
210 bool data_from_disk_cache =
false;
211 bool data_to_disk_cache =
false;
214 bool use_disk_cache = valid_disk_cache();
218 if(
true == use_disk_cache) {
220 BESDEBUG(
"h5",
"Coming to use disk cache "<<endl);
222 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
223 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
224 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
226 string cache_fname=HDF5CFUtil::obtain_cache_fname(diskcache_prefix,filename,varname);
227 cache_fpath = diskcache_dir +
"/"+ cache_fname;
229 int temp_total_elems = 1;
230 for (
unsigned int i = 0; i <dimsizes.size();i++)
231 temp_total_elems = temp_total_elems*dimsizes[i];
232 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
234 if(is_dap4 ==
false && dtype==H5CHAR)
237 int expected_file_size = dtype_size *temp_total_elems;
240 if(
true == disk_cache->get_data_from_cache(cache_fpath, expected_file_size,fd)) {
242 vector<size_t> offset_size_t;
243 offset_size_t.resize(rank);
244 for(
int i = 0; i <rank;i++)
245 offset_size_t[i] = (
size_t)offset[i];
246 size_t offset_1st = INDEX_nD_TO_1D(dimsizes,offset_size_t);
249 for (
int i = 0; i < rank; i++)
250 end[i] = offset[i] +(count[i]-1)*step[i];
251 size_t offset_last = INDEX_nD_TO_1D(dimsizes,end);
256 size_t total_read = dtype_size*(offset_last-offset_1st+1);
258 off_t fpos = lseek(fd,dtype_size*offset_1st,SEEK_SET);
266 data_from_disk_cache = obtain_cached_data(disk_cache,cache_fpath,fd, step,count,total_read,dtype_size);
270 if(
true == data_from_disk_cache)
273 data_to_disk_cache =
true;
279 bool pass_fileid = HDF5RequestHandler::get_pass_fileid();
280 if(
false == pass_fileid) {
281 if ((fileid = H5Fopen(filename.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT))<0) {
283 eherr <<
"HDF5 File " << filename
284 <<
" cannot be opened. "<<endl;
285 throw InternalErr (__FILE__, __LINE__, eherr.str ());
289 if ((dsetid = H5Dopen(fileid,varname.c_str(),H5P_DEFAULT))<0) {
290 HDF5CFUtil::close_fileid(fileid,pass_fileid);
292 eherr <<
"HDF5 dataset " << varname
293 <<
" cannot be opened. "<<endl;
294 throw InternalErr (__FILE__, __LINE__, eherr.str ());
297 if ((dspace = H5Dget_space(dsetid))<0) {
300 HDF5CFUtil::close_fileid(fileid,pass_fileid);
302 eherr <<
"Space id of the HDF5 dataset " << varname
303 <<
" cannot be obtained. "<<endl;
304 throw InternalErr (__FILE__, __LINE__, eherr.str ());
307 if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
308 hoffset.data(), hstep.data(),
309 hcount.data(),
nullptr) < 0) {
313 HDF5CFUtil::close_fileid(fileid,pass_fileid);
315 eherr <<
"The selection of hyperslab of the HDF5 dataset " << varname
317 throw InternalErr (__FILE__, __LINE__, eherr.str ());
320 mspace = H5Screate_simple(rank, hcount.data(),
nullptr);
324 HDF5CFUtil::close_fileid(fileid,pass_fileid);
326 eherr <<
"The creation of the memory space of the HDF5 dataset " << varname
328 throw InternalErr (__FILE__, __LINE__, eherr.str ());
332 if ((dtypeid = H5Dget_type(dsetid)) < 0) {
337 HDF5CFUtil::close_fileid(fileid,pass_fileid);
339 eherr <<
"Obtaining the datatype of the HDF5 dataset " << varname
341 throw InternalErr (__FILE__, __LINE__, eherr.str ());
345 if ((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
351 HDF5CFUtil::close_fileid(fileid,pass_fileid);
353 eherr <<
"Obtaining the memory type of the HDF5 dataset " << varname
355 throw InternalErr (__FILE__, __LINE__, eherr.str ());
363 if(
true == add_mem_cache) {
369 HDF5CFUtil::close_fileid(fileid,pass_fileid);
370 throw InternalErr(__FILE__,__LINE__,
"The memory data cache buffer needs to be set");
372 read_ret= H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf);
378 HDF5CFUtil::close_fileid(fileid,pass_fileid);
379 throw InternalErr(__FILE__,__LINE__,
"Cannot read the data to the buffer.");
395 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
403 HDF5CFUtil::close_fileid(fileid,pass_fileid);
405 eherr <<
"Cannot read the HDF5 dataset " << varname
406 <<
" with the type of H5T_NATIVE_CHAR "<<endl;
407 throw InternalErr (__FILE__, __LINE__, eherr.str ());
412 set_value((dods_int8 *)val.data(),nelms);
416 newval.resize(nelms);
418 for (
int counter = 0; counter < nelms; counter++)
419 newval[counter] = (
short) (val[counter]);
421 set_value ((dods_int16 *) newval.data(), nelms);
424 if(
true == data_to_disk_cache) {
426 BESDEBUG(
"h5",
"writing data to disk cache "<<endl);
427 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,2,val,nelms);
435 HDF5CFUtil::close_fileid(fileid,pass_fileid);
437 eherr <<
"write data to cache failed.";
438 throw InternalErr (__FILE__, __LINE__, eherr.str ());
459 size_t dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
461 val.resize(nelms*dtype_size);
463 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
470 HDF5CFUtil::close_fileid(fileid,pass_fileid);
472 eherr <<
"Cannot read the HDF5 dataset " << varname
473 <<
" with the type of H5T_NATIVE_UCHAR "<<endl;
474 throw InternalErr (__FILE__, __LINE__, eherr.str ());
481 if(
true == data_to_disk_cache) {
482 BESDEBUG(
"h5",
"writing data to disk cache "<<endl);
484 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,dtype_size,val,nelms);
492 HDF5CFUtil::close_fileid(fileid,pass_fileid);
494 eherr <<
"Write data to cache failed."
495 <<
"It is very possible the error is caused by the server failure"
496 <<
" such as filled disk partition at the server rather than Hyrax. Please contact "
497 <<
" the corresponding data center first. If the issue is not due to "
499 throw InternalErr (__FILE__, __LINE__, eherr.str ());
515 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
523 HDF5CFUtil::close_fileid(fileid,pass_fileid);
526 eherr <<
"Cannot read the HDF5 dataset " << varname
527 <<
" with the type of H5T_NATIVE_SHORT "<<endl;
528 throw InternalErr (__FILE__, __LINE__, eherr.str ());
531 set_value ((dods_int16 *) val.data(), nelms);
538 vector<unsigned short> val;
540 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
548 HDF5CFUtil::close_fileid(fileid,pass_fileid);
550 eherr <<
"Cannot read the HDF5 dataset " << varname
551 <<
" with the type of H5T_NATIVE_USHORT "<<endl;
552 throw InternalErr (__FILE__, __LINE__, eherr.str ());
555 set_value ((dods_uint16 *) val.data(), nelms);
564 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
571 HDF5CFUtil::close_fileid(fileid,pass_fileid);
573 eherr <<
"Cannot read the HDF5 dataset " << varname
574 <<
" with the type of H5T_NATIVE_INT "<<endl;
575 throw InternalErr (__FILE__, __LINE__, eherr.str ());
578 set_value ((dods_int32 *) val.data(), nelms);
584 vector<unsigned int>val;
586 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
593 HDF5CFUtil::close_fileid(fileid,pass_fileid);
595 eherr <<
"Cannot read the HDF5 dataset " << varname
596 <<
" with the type of H5T_NATIVE_UINT "<<endl;
597 throw InternalErr (__FILE__, __LINE__, eherr.str ());
600 set_value ((dods_uint32 *) val.data(), nelms);
610 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
617 HDF5CFUtil::close_fileid(fileid,pass_fileid);
619 eherr <<
"Cannot read the HDF5 dataset " << varname
620 <<
" with the type of H5T_NATIVE_FLOAT "<<endl;
621 throw InternalErr (__FILE__, __LINE__, eherr.str ());
624 set_value ((dods_float32 *) val.data(), nelms);
634 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
642 HDF5CFUtil::close_fileid(fileid,pass_fileid);
644 eherr <<
"Cannot read the HDF5 dataset " << varname
645 <<
" with the type of H5T_NATIVE_DOUBLE "<<endl;
646 throw InternalErr (__FILE__, __LINE__, eherr.str ());
649 set_value ((dods_float64 *) val.data(), nelms);
657 size_t ty_size = H5Tget_size(dtypeid);
664 HDF5CFUtil::close_fileid(fileid,pass_fileid);
666 eherr <<
"Cannot obtain the size of the fixed size HDF5 string of the dataset "
668 throw InternalErr (__FILE__, __LINE__, eherr.str ());
671 vector <char> strval;
672 strval.resize(nelms*ty_size);
673 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)strval.data());
681 HDF5CFUtil::close_fileid(fileid,pass_fileid);
683 eherr <<
"Cannot read the HDF5 dataset " << varname
684 <<
" with the type of the fixed size HDF5 string "<<endl;
685 throw InternalErr (__FILE__, __LINE__, eherr.str ());
688 string total_string(strval.begin(),strval.end());
690 vector <string> finstrval;
691 finstrval.resize(nelms);
692 for (
int i = 0; i<nelms; i++)
693 finstrval[i] = total_string.substr(i*ty_size,ty_size);
700 if ((
true == HDF5RequestHandler::get_drop_long_string()) &&
701 ty_size > NC_JAVA_STR_SIZE_LIMIT) {
702 for (
int i = 0; i<nelms; i++)
705 set_value(finstrval,nelms);
706 total_string.clear();
713 size_t ty_size = H5Tget_size(memtype);
720 HDF5CFUtil::close_fileid(fileid,pass_fileid);
722 eherr <<
"Cannot obtain the size of the fixed size HDF5 string of the dataset "
724 throw InternalErr (__FILE__, __LINE__, eherr.str ());
726 vector <char> strval;
727 strval.resize(nelms*ty_size);
728 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)strval.data());
736 HDF5CFUtil::close_fileid(fileid,pass_fileid);
738 eherr <<
"Cannot read the HDF5 dataset " << varname
739 <<
" with the type of the HDF5 variable length string "<<endl;
740 throw InternalErr (__FILE__, __LINE__, eherr.str ());
743 vector<string>finstrval;
744 finstrval.resize(nelms);
745 char*temp_bp = strval.data();
746 char*onestring =
nullptr;
747 for (
int i =0;i<nelms;i++) {
748 onestring = *(
char**)temp_bp;
749 if(onestring!=
nullptr )
750 finstrval[i] =string(onestring);
757 if (
false == strval.empty()) {
758 herr_t ret_vlen_claim;
759 ret_vlen_claim = H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(
void*)strval.data());
760 if (ret_vlen_claim < 0){
766 HDF5CFUtil::close_fileid(fileid,pass_fileid);
768 eherr <<
"Cannot reclaim the memory buffer of the HDF5 variable length string of the dataset "
770 throw InternalErr (__FILE__, __LINE__, eherr.str ());
778 if (
true == HDF5RequestHandler::get_drop_long_string()) {
779 bool drop_long_str =
false;
780 for (
int i =0;i<nelms;i++) {
781 if(finstrval[i].size() >NC_JAVA_STR_SIZE_LIMIT){
782 drop_long_str =
true;
786 if (drop_long_str ==
true) {
787 for (
int i =0;i<nelms;i++)
791 set_value(finstrval,nelms);
803 HDF5CFUtil::close_fileid(fileid,pass_fileid);
805 eherr <<
"Cannot read the HDF5 dataset " << varname
806 <<
" with the unsupported HDF5 datatype"<<endl;
807 throw InternalErr (__FILE__, __LINE__, eherr.str ());
816 HDF5CFUtil::close_fileid(fileid,pass_fileid);
821bool HDF5CFArray::valid_disk_cache() {
823 bool ret_value =
false;
824 if(
true == HDF5RequestHandler::get_use_disk_cache()) {
826 BESDEBUG(
"h5",
"Coming to disk cache "<<endl);
828 if(dtype == H5CHAR || dtype ==H5UCHAR || dtype==H5INT16 || dtype ==H5UINT16 ||
829 dtype == H5INT32 || dtype ==H5UINT32 || dtype ==H5FLOAT32 || dtype==H5FLOAT64 ||
830 dtype == H5INT64 || dtype ==H5UINT64){
832 BESDEBUG(
"h5",
"Coming to disk cache datatype block"<<endl);
834 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
835 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
836 long diskcache_size = HDF5RequestHandler::get_disk_cache_size();
838 if((
"" == diskcache_dir)||(
""==diskcache_prefix)||(diskcache_size <=0))
839 throw InternalErr (__FILE__, __LINE__,
"Either the cached dir is empty or the prefix is nullptr or the cache size is not set.");
842 if(stat(diskcache_dir.c_str(),&sb) !=0) {
843 string err_mesg=
"The cached directory " + diskcache_dir;
844 err_mesg = err_mesg +
" doesn't exist. ";
845 throw InternalErr(__FILE__,__LINE__,err_mesg);
848 if(
true == S_ISDIR(sb.st_mode)) {
849 if(access(diskcache_dir.c_str(),R_OK|W_OK|X_OK) == -1) {
850 string err_mesg=
"The cached directory " + diskcache_dir;
851 err_mesg = err_mesg +
" can NOT be read,written or executable.";
852 throw InternalErr(__FILE__,__LINE__,err_mesg);
856 string err_mesg=
"The cached directory " + diskcache_dir;
857 err_mesg = err_mesg +
" is not a directory.";
858 throw InternalErr(__FILE__,__LINE__,err_mesg);
863 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
865 if(
true == HDF5RequestHandler::get_disk_cache_comp_data()){
866 BESDEBUG(
"h5",
"Compression disk cache key is true"<<endl);
867 ret_value = valid_disk_cache_for_compressed_data(dtype_size);
868 BESDEBUG(
"h5",
"variable disk cache passes the compression parameter check"<<endl);
871 BESDEBUG(
"h5",
"Compression disk cache key is NOT set, disk cache key is true."<<endl);
881bool HDF5CFArray:: valid_disk_cache_for_compressed_data(
short dtype_size)
const {
883 bool ret_value =
false;
887 size_t total_byte = total_elems*dtype_size;
889 if((comp_ratio < HDF5RequestHandler::get_disk_comp_threshold())
890 && (total_elems*dtype_size >= HDF5RequestHandler::get_disk_var_size())) {
891 if(
true == HDF5RequestHandler::get_disk_cache_float_only_comp()) {
892 if(dtype==H5FLOAT32 || dtype == H5FLOAT64)
902bool HDF5CFArray::obtain_cached_data(
HDF5DiskCache *disk_cache,
const string & cache_fpath,
int fd,vector<int> &cd_step, vector<int>&cd_count,
size_t total_read,
short dtype_size) {
904 ssize_t ret_read_val = -1;
907 buf.resize(total_read);
910 if((-1 == ret_read_val) || (ret_read_val != (ssize_t)total_read)) {
915 unsigned int nele_to_read = 1;
916 for(
int i = 0; i<rank;i++)
917 nele_to_read *=cd_count[i];
919 if(nele_to_read == (total_read/dtype_size)) {
925 vector<int>cd_start(rank,0);
926 vector<size_t>cd_pos(rank,0);
927 int nelms_to_send = 1;
928 for(
int i = 0; i <rank; i++)
929 nelms_to_send = nelms_to_send*cd_count[i];
936 vector<int>total_val;
937 total_val.resize(total_read/dtype_size);
938 memcpy(total_val.data(),(
void*)buf.data(),total_read);
940 vector<int>final_val;
955 if(is_dap4 ==
false) {
956 vector<short>final_val;
968 set_value((dods_int16*)final_val.data(),nelms_to_send);
971 vector<char>final_val;
983 set_value((dods_int8*)final_val.data(),nelms_to_send);
992 vector<unsigned char>total_val;
993 total_val.resize(total_read/dtype_size);
994 memcpy(total_val.data(),(
void*)buf.data(),total_read);
996 vector<unsigned char>final_val;
997 subset<unsigned char>(
1010 vector<unsigned char>final_val;
1011 subset<unsigned char>(
1023 set_value ((dods_byte *) final_val.data(), nelms_to_send);
1030 vector<short>total_val;
1031 total_val.resize(total_read/dtype_size);
1032 memcpy(total_val.data(),(
void*)buf.data(),total_read);
1034 vector<short>final_val;
1048 vector<short>final_val;
1061 set_value ((dods_int16 *) final_val.data(), nelms_to_send);
1068 vector<unsigned short>total_val;
1069 total_val.resize(total_read/dtype_size);
1070 memcpy(total_val.data(),(
void*)buf.data(),total_read);
1072 vector<unsigned short>final_val;
1073 subset<unsigned short>(
1086 vector<unsigned short>final_val;
1087 subset<unsigned short>(
1099 set_value ((dods_uint16 *) final_val.data(), nelms_to_send);
1106 vector<int>total_val;
1107 total_val.resize(total_read/dtype_size);
1108 memcpy(total_val.data(),(
void*)buf.data(),total_read);
1110 vector<int>final_val;
1125 vector<int>final_val;
1139 set_value ((dods_int32 *) final_val.data(), nelms_to_send);
1146 vector<unsigned int>total_val;
1147 total_val.resize(total_read/dtype_size);
1148 memcpy(total_val.data(),(
void*)buf.data(),total_read);
1150 vector<unsigned int>final_val;
1151 subset<unsigned int>(
1164 vector<unsigned int>final_val;
1165 subset<unsigned int>(
1177 set_value ((dods_uint32 *) final_val.data(), nelms_to_send);
1184 vector<unsigned int>total_val;
1185 total_val.resize(total_read/dtype_size);
1186 memcpy(total_val.data(),(
void*)buf.data(),total_read);
1188 vector<unsigned int>final_val;
1189 subset<unsigned int>(
1202 vector<long long >final_val;
1215 set_value ((dods_int64 *) final_val.data(), nelms_to_send);
1224 vector<unsigned int>total_val;
1225 total_val.resize(total_read/dtype_size);
1226 memcpy(total_val.data(),(
void*)buf.data(),total_read);
1228 vector<unsigned int>final_val;
1229 subset<unsigned int>(
1242 vector<unsigned long long >final_val;
1243 subset<unsigned long long >(
1255 set_value ((dods_uint64 *) final_val.data(), nelms_to_send);
1263 vector<float>total_val;
1264 total_val.resize(total_read/dtype_size);
1265 memcpy(total_val.data(),(
void*)buf.data(),total_read);
1267 vector<float>final_val;
1281 vector<float>final_val;
1295 set_value ((dods_float32 *) final_val.data(), nelms_to_send);
1301 vector<double>total_val;
1302 total_val.resize(total_read/dtype_size);
1303 memcpy(total_val.data(),(
void*)buf.data(),total_read);
1305 vector<double>final_val;
1318 vector<double>final_val;
1331 set_value ((dods_float64 *) final_val.data(), nelms_to_send);
1335 throw InternalErr (__FILE__, __LINE__,
"unsupported data type.");
1345HDF5CFArray::write_data_to_cache(hid_t dset_id, hid_t , hid_t , hid_t memtype,
1346 const string& cache_fpath,
short dtype_size,
const vector<char> &buf,
int nelms) {
1348 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
1349 string disk_cache_dir = HDF5RequestHandler::get_disk_cache_dir();
1350 string disk_cache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
1352 int total_nelem = 1;
1353 for(
int i = 0; i <rank; i++)
1354 total_nelem = total_nelem*dimsizes[i];
1358 if(H5CHAR == dtype && is_dap4 ==
false) {
1360 vector<short>newval;
1361 newval.resize(total_nelem);
1362 if(total_nelem == nelms) {
1363 for (
int i = 0; i < total_nelem;i++)
1364 newval[i] = (
short)buf[i];
1365 disk_cache->write_cached_data2(cache_fpath,
sizeof(
short)*total_nelem,(
const void*)newval.data());
1369 val2.resize(total_nelem);
1370 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, val2.data())<0)
1371 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole HDF5 dataset for the disk cache.");
1372 for (
int i = 0; i < total_nelem;i++)
1373 newval[i] = (
short)val2[i];
1374 disk_cache->write_cached_data2(cache_fpath,
sizeof(
short)*total_nelem,(
const void*)newval.data());
1378 if(total_nelem == nelms) {
1379 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)buf.data());
1382 val.resize(dtype_size*total_nelem);
1383 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, val.data())<0)
1384 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole SDS for cache.");
1386 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)val.data());
1392void HDF5CFArray::read_data_from_mem_cache(
void*buf) {
1397 int nelms = format_constraint (offset.data(), step.data(), count.data());
1399 vector<int>at_pos(at_ndims,0);
1400 for (
int i = 0; i< rank; i++)
1401 at_pos[i] = at_offset[i];
1409 vector<unsigned char> val;
1411 subset<unsigned char>(
1424 set_value ((dods_byte *) val.data(), nelms);
1436 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,val.data());
1438 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
1448 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1449 ostringstream eherr;
1450 eherr <<
"Cannot read the HDF5 dataset " << varname
1451 <<
" with the type of H5T_NATIVE_CHAR "<<endl;
1452 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1456 vector<short>newval;
1457 newval.resize(nelms);
1459 for (
int counter = 0; counter < nelms; counter++)
1460 newval[counter] = (
short) (val[counter]);
1462 set_value ((dods_int16 *) newval.data(), nelms);
1473 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,val.data());
1475 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
1485 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1487 ostringstream eherr;
1488 eherr <<
"Cannot read the HDF5 dataset " << varname
1489 <<
" with the type of H5T_NATIVE_SHORT "<<endl;
1490 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1493 set_value ((dods_int16 *) val.data(), nelms);
1500 vector<unsigned short> val;
1503 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,val.data());
1505 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
1509 if (rank > 0) H5Sclose(mspace);
1514 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1515 ostringstream eherr;
1516 eherr <<
"Cannot read the HDF5 dataset " << varname
1517 <<
" with the type of H5T_NATIVE_USHORT "<<endl;
1518 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1521 set_value ((dods_uint16 *) val.data(), nelms);
1531 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,val.data());
1533 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
1542 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1543 ostringstream eherr;
1544 eherr <<
"Cannot read the HDF5 dataset " << varname
1545 <<
" with the type of H5T_NATIVE_INT "<<endl;
1546 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1549 set_value ((dods_int32 *) val.data(), nelms);
1555 vector<unsigned int>val;
1558 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,val.data());
1560 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
1570 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1571 ostringstream eherr;
1572 eherr <<
"Cannot read the HDF5 dataset " << varname
1573 <<
" with the type of H5T_NATIVE_UINT "<<endl;
1574 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1577 set_value ((dods_uint32 *) val.data(), nelms);
1588 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,val.data());
1590 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
1599 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1600 ostringstream eherr;
1601 eherr <<
"Cannot read the HDF5 dataset " << varname
1602 <<
" with the type of H5T_NATIVE_FLOAT "<<endl;
1603 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1606 set_value ((dods_float32 *) val.data(), nelms);
1617 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,val.data());
1619 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,val.data());
1628 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1629 ostringstream eherr;
1630 eherr <<
"Cannot read the HDF5 dataset " << varname
1631 <<
" with the type of H5T_NATIVE_DOUBLE "<<endl;
1632 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1635 set_value ((dods_float64 *) val.data(), nelms);
1650BaseType* HDF5CFArray::h5cfdims_transform_to_dap4(D4Group *grp) {
1654 Array *dest =
static_cast<HDF5CFArray*
>(ptr_duplicate());
1660 D4Dimensions *grp_dims = grp->dims();
1661 for (Array::Dim_iter dap2_dim = dest->dim_begin(), e = dest->dim_end(); dap2_dim != e; ++dap2_dim) {
1662 if (!(*dap2_dim).name.empty()) {
1665 D4Dimension *d4_dim = grp_dims->find_dim((*dap2_dim).name);
1667 d4_dim =
new D4Dimension((*dap2_dim).name, (*dap2_dim).size);
1668 grp_dims->add_dim_nocopy(d4_dim);
1672 (*dap2_dim).dim = d4_dim;
1684BaseType* HDF5CFArray::h5cfdims_transform_to_dap4_int64(D4Group *grp) {
1688 Array *dest =
static_cast<HDF5CFArray*
>(ptr_duplicate());
1694 for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1695 if (
false == (*d).name.empty()) {
1697 D4Group *temp_grp = grp;
1698 D4Dimension *d4_dim =
nullptr;
1701 D4Dimensions *temp_dims = temp_grp->dims();
1704 d4_dim = temp_dims->find_dim((*d).name);
1710 if(temp_grp->get_parent())
1711 temp_grp =
static_cast<D4Group*
>(temp_grp->get_parent());
1720 bool d4_dim_null = ((d4_dim==
nullptr)?
true:
false);
1725 if(d4_dim_null ==
true) {
1727 d4_dim =
new D4Dimension((*d).name, (*d).size);
1728 D4Dimensions * dims = grp->dims();
1729 dims->add_dim_nocopy(d4_dim);
1735 dest->set_is_dap4(
true);
1744HDF5CFArray::format_constraint (
int *offset,
int *step,
int *count)
1750 Dim_iter p = dim_begin ();
1752 while (p != dim_end ()) {
1754 int start = dimension_start (p,
true);
1755 int stride = dimension_stride (p,
true);
1756 int stop = dimension_stop (p,
true);
1762 oss <<
"Array/Grid hyperslab start point "<< start <<
1763 " is greater than stop point " << stop <<
".";
1764 throw Error(malformed_expr, oss.str());
1769 count[id] = ((stop - start) / stride) + 1;
1773 "=format_constraint():"
1774 <<
"id=" <<
id <<
" offset=" << offset[
id]
1775 <<
" step=" << step[
id]
1776 <<
" count=" << count[
id]
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
include the entry functions to execute the handlers
virtual void unlock_and_close(const std::string &target)
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
void read_data_NOT_from_mem_cache(bool add_cache, void *buf) override
static HDF5DiskCache * get_instance(const long, const std::string &, const std::string &)
Helper functions for generating DAS attributes and a function to check BES Key.
static ssize_t read_buffer_from_file(int fd, void *buf, size_t)
Getting a subset of a variable.