44#include <libdap/DMR.h>
45#include <libdap/D4BaseTypeFactory.h>
46#include <BESDMRResponse.h>
47#include <ObjMemCache.h>
50#include <libdap/mime_util.h>
55#include <BESDASResponse.h>
56#include <BESDDSResponse.h>
57#include <BESDataDDSResponse.h>
58#include <libdap/Ancillary.h>
60#include <BESDapNames.h>
61#include <BESResponseNames.h>
62#include <BESContainer.h>
63#include <BESResponseHandler.h>
64#include <BESVersionInfo.h>
65#include <BESServiceRegistry.h>
67#include <BESDapError.h>
68#include <BESInternalFatalError.h>
69#include <TheBESKeys.h>
71#include <BESStopWatch.h>
73#include "config_hdf5.h"
84#define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
87void get_attr_contents(AttrTable* temp_table);
91void write_das_to_file(DAS*das_ptr,FILE* das_file);
94void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
97void write_container_name_to_file(
const string&,FILE* das_file);
100void write_das_attr_info(AttrTable* dtp,
const string&,
const string&,FILE * das_file);
103char* copy_str(
char *temp_ptr,
const string & str);
107char* obtain_str(
char*temp_ptr,
string & str);
110char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at);
118static unsigned int get_uint_key(
const string &key,
unsigned int def_val);
119static unsigned long get_ulong_key(
const string &key,
unsigned long def_val);
122static float get_float_key(
const string &key,
float def_val);
125static string get_beskeys(
const string&);
131bool obtain_beskeys_info(
const string &,
bool &);
134extern void read_cfdas(DAS &das,
const string & filename,hid_t fileid);
135extern void read_cfdds(DDS &dds,
const string & filename,hid_t fileid);
136extern void read_cfdmr(DMR *dmr,
const string & filename,hid_t fileid);
141unsigned int HDF5RequestHandler::_mdcache_entries = 500;
142unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
143unsigned int HDF5RequestHandler::_srdcache_entries = 0;
144float HDF5RequestHandler::_cache_purge_level = 0.2F;
147ObjMemCache *HDF5RequestHandler::das_cache =
nullptr;
148ObjMemCache *HDF5RequestHandler::dds_cache =
nullptr;
149ObjMemCache *HDF5RequestHandler::datadds_cache =
nullptr;
150ObjMemCache *HDF5RequestHandler::dmr_cache =
nullptr;
152ObjMemCache *HDF5RequestHandler::lrdata_mem_cache =
nullptr;
153ObjMemCache *HDF5RequestHandler::srdata_mem_cache =
nullptr;
159bool HDF5RequestHandler::_usecf =
true;
160bool HDF5RequestHandler::_pass_fileid =
false;
161bool HDF5RequestHandler::_disable_structmeta =
true;
162bool HDF5RequestHandler::_disable_ecsmeta =
false;
163bool HDF5RequestHandler::_keep_var_leading_underscore =
false;
164bool HDF5RequestHandler::_check_name_clashing =
false;
165bool HDF5RequestHandler::_add_path_attrs =
true;
166bool HDF5RequestHandler::_drop_long_string =
true;
167bool HDF5RequestHandler::_fillvalue_check =
true;
168bool HDF5RequestHandler::_check_ignore_obj =
false;
169bool HDF5RequestHandler::_flatten_coor_attr =
true;
170bool HDF5RequestHandler::_default_handle_dimension =
true;
171bool HDF5RequestHandler::_eos5_rm_convention_attr_path =
true;
172bool HDF5RequestHandler::_dmr_long_int =
true;
173bool HDF5RequestHandler::_no_zero_size_fullnameattr =
false;
174bool HDF5RequestHandler::_enable_coord_attr_add_path =
true;
176bool HDF5RequestHandler::_usecfdmr =
true;
177bool HDF5RequestHandler::_add_dap4_coverage =
true;
179bool HDF5RequestHandler::_common_cache_dirs =
false;
181bool HDF5RequestHandler::_use_disk_cache =
false;
182bool HDF5RequestHandler::_use_disk_dds_cache =
false;
183string HDF5RequestHandler::_disk_cache_dir =
"";
184string HDF5RequestHandler::_disk_cachefile_prefix =
"";
185unsigned long long HDF5RequestHandler::_disk_cache_size =0;
188bool HDF5RequestHandler::_disk_cache_comp_data =
false;
189bool HDF5RequestHandler::_disk_cache_float_only_comp_data =
false;
190float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
191unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
193bool HDF5RequestHandler::_use_disk_meta_cache =
false;
194string HDF5RequestHandler::_disk_meta_cache_path =
"";
196bool HDF5RequestHandler::_use_latlon_disk_cache =
false;
197long HDF5RequestHandler::_latlon_disk_cache_size =0;
198string HDF5RequestHandler::_latlon_disk_cache_dir =
"";
199string HDF5RequestHandler::_latlon_disk_cachefile_prefix=
"";
202bool HDF5RequestHandler::_escape_utf8_attr =
true;
204DMR* HDF5RequestHandler::dmr_int64 =
nullptr;
211string HDF5RequestHandler::_stp_east_filename;
212string HDF5RequestHandler::_stp_north_filename;
213vector<string> HDF5RequestHandler::lrd_cache_dir_list;
214vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
215vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
221HDF5RequestHandler::HDF5RequestHandler(
const string & name)
225 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
227 add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
228 add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
229 add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
230 add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
231 add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
233 add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
234 add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
236#if !(DYNAMIC_CONFIG_ENABLED)
240 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
243HDF5RequestHandler::~HDF5RequestHandler()
248 delete datadds_cache;
250 delete lrdata_mem_cache;
251 delete srdata_mem_cache;
258void HDF5RequestHandler::load_config()
260 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
262 if (BESISDEBUG(HDF5_NAME)){
263 sw.
start(prolog,
"ClockTheBESKeys");
267 HDF5RequestHandler::_mdcache_entries = get_uint_key(
"H5.MetaDataMemCacheEntries", 0);
268 HDF5RequestHandler::_lrdcache_entries = get_uint_key(
"H5.LargeDataMemCacheEntries", 0);
269 HDF5RequestHandler::_srdcache_entries = get_uint_key(
"H5.SmallDataMemCacheEntries", 0);
270 HDF5RequestHandler::_cache_purge_level = get_float_key(
"H5.CachePurgeLevel", 0.2F);
272 if (get_mdcache_entries()) {
273 das_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
274 dds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
275 datadds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
276 dmr_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
288 bool has_key =
false;
289 bool key_value = obtain_beskeys_info(
"H5.EnableCF",has_key);
292 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCF: " << (_usecf?
"true":
"false") << endl);
295 key_value = obtain_beskeys_info(
"H5.DefaultHandleDimension",has_key);
297 _default_handle_dimension = key_value;
298 BESDEBUG(HDF5_NAME, prolog <<
"H5.DefaultHandleDimension: " << (_default_handle_dimension?
"true":
"false") << endl);
301 key_value = obtain_beskeys_info(
"H5.EnablePassFileID",has_key);
303 _pass_fileid = key_value;
304 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnablePassFileID: " << (_pass_fileid?
"true":
"false") << endl);
306 key_value = obtain_beskeys_info(
"H5.DisableStructMetaAttr",has_key);
308 _disable_structmeta = key_value;
309 BESDEBUG(HDF5_NAME, prolog <<
"H5.DisableStructMetaAttr: " << (_disable_structmeta?
"true":
"false") << endl);
311 key_value = obtain_beskeys_info(
"H5.DisableECSMetaAttr",has_key);
313 _disable_ecsmeta = key_value;
314 BESDEBUG(HDF5_NAME, prolog <<
"H5.DisableECSMetaAttr: " << (_disable_ecsmeta?
"true":
"false") << endl);
316 key_value = obtain_beskeys_info(
"H5.KeepVarLeadingUnderscore",has_key);
318 _keep_var_leading_underscore = key_value;
319 BESDEBUG(HDF5_NAME, prolog <<
"H5.KeepVarLeadingUnderscore: " << (_keep_var_leading_underscore?
"true":
"false") << endl);
324 key_value = obtain_beskeys_info(
"H5.EnableCheckNameClashing",has_key);
326 _check_name_clashing = key_value;
327 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCheckNameClashing: " << (_check_name_clashing?
"true":
"false") << endl);
329 key_value = obtain_beskeys_info(
"H5.EnableAddPathAttrs",has_key);
331 _add_path_attrs = key_value;
332 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableAddPathAttrs: " << (_add_path_attrs?
"true":
"false") << endl);
334 key_value = obtain_beskeys_info(
"H5.EnableDropLongString",has_key);
336 _drop_long_string = key_value;
337 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDropLongString: " << (_drop_long_string?
"true":
"false") << endl);
339 key_value = obtain_beskeys_info(
"H5.EnableFillValueCheck",has_key);
341 _fillvalue_check = key_value;
342 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableFillValueCheck: " << (_fillvalue_check?
"true":
"false") << endl);
345 key_value = obtain_beskeys_info(
"H5.CheckIgnoreObj",has_key);
347 _check_ignore_obj = key_value;
348 BESDEBUG(HDF5_NAME, prolog <<
"H5.CheckIgnoreObj: " << (_check_ignore_obj?
"true":
"false") << endl);
350 key_value = obtain_beskeys_info(
"H5.ForceFlattenNDCoorAttr",has_key);
352 _flatten_coor_attr = key_value;
353 BESDEBUG(HDF5_NAME, prolog <<
"H5.ForceFlattenNDCoorAttr: " << (_flatten_coor_attr?
"true":
"false") << endl);
356 key_value = obtain_beskeys_info(
"H5.RmConventionAttrPath",has_key);
358 _eos5_rm_convention_attr_path = key_value;
359 BESDEBUG(HDF5_NAME, prolog <<
"H5.RmConventionAttrPath: " << (_eos5_rm_convention_attr_path?
"true":
"false") << endl);
361 key_value = obtain_beskeys_info(
"H5.EnableDMR64bitInt",has_key);
363 _dmr_long_int = key_value;
364 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDMR64bitInt: " << (_dmr_long_int?
"true":
"false") << endl);
366 key_value = obtain_beskeys_info(
"H5.NoZeroSizeFullnameAttr",has_key);
368 _no_zero_size_fullnameattr = key_value;
369 BESDEBUG(HDF5_NAME, prolog <<
"H5.NoZeroSizeFullnameAttr: " << (_no_zero_size_fullnameattr?
"true":
"false") << endl);
371 key_value = obtain_beskeys_info(
"H5.EnableCoorattrAddPath",has_key);
373 _enable_coord_attr_add_path = key_value;
374 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCoorattrAddPath: " << (_enable_coord_attr_add_path?
"true":
"false") << endl);
376 key_value = obtain_beskeys_info(
"H5.EnableCFDMR",has_key);
378 _usecfdmr = key_value;
379 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCFDMR: " << (_usecfdmr?
"true":
"false") << endl);
381 key_value = obtain_beskeys_info(
"H5.EnableDAP4Coverage",has_key);
383 _add_dap4_coverage = key_value;
384 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDAP4Coverage: " << (_add_dap4_coverage?
"true":
"false") << endl);
386 key_value = obtain_beskeys_info(
"H5.EnableDiskDataCache",has_key);
388 _use_disk_cache = key_value;
389 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDiskDataCache: " << (_use_disk_cache?
"true":
"false") << endl);
390 _disk_cache_dir = get_beskeys(
"H5.DiskCacheDataPath");
391 _disk_cachefile_prefix = get_beskeys(
"H5.DiskCacheFilePrefix");
392 _disk_cache_size = get_ulong_key(
"H5.DiskCacheSize",0);
394 key_value = obtain_beskeys_info(
"H5.DiskCacheComp",has_key);
396 _disk_cache_comp_data = key_value;
397 BESDEBUG(HDF5_NAME, prolog <<
"H5.DiskCacheComp: " << (_disk_cache_comp_data?
"true":
"false") << endl);
399 key_value = obtain_beskeys_info(
"H5.DiskCacheFloatOnlyComp",has_key);
401 _disk_cache_float_only_comp_data = key_value;
402 BESDEBUG(HDF5_NAME, prolog <<
"H5.DiskCacheFloatOnlyComp: " << (_disk_cache_float_only_comp_data?
"true":
"false") << endl);
403 _disk_cache_comp_threshold = get_float_key(
"H5.DiskCacheCompThreshold",1.0);
404 _disk_cache_var_size = 1024*get_uint_key(
"H5.DiskCacheCompVarSize",0);
406 key_value = obtain_beskeys_info(
"H5.EnableDiskMetaDataCache",has_key);
408 _use_disk_meta_cache = key_value;
409 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDiskMetaDataCache: " << (_use_disk_meta_cache?
"true":
"false") << endl);
411 key_value = obtain_beskeys_info(
"H5.EnableDiskDDSCache",has_key);
413 _use_disk_dds_cache = key_value;
414 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDiskDDSCache: " << (_use_disk_dds_cache?
"true":
"false") << endl);
415 _disk_meta_cache_path = get_beskeys(
"H5.DiskMetaDataCachePath");
417 key_value = obtain_beskeys_info(
"H5.EnableEOSGeoCacheFile",has_key);
419 _use_latlon_disk_cache = key_value;
420 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableEOSGeoCacheFile: " << (_use_latlon_disk_cache?
"true":
"false") << endl);
421 _latlon_disk_cache_size = get_uint_key(
"H5.Cache.latlon.size",0);
422 _latlon_disk_cache_dir = get_beskeys(
"H5.Cache.latlon.path");
423 _latlon_disk_cachefile_prefix= get_beskeys(
"H5.Cache.latlon.prefix");
425 key_value = obtain_beskeys_info(
"H5.EscapeUTF8Attr", has_key);
427 _escape_utf8_attr = key_value;
431 key_value = obtain_beskeys_info(
"H5.DefaultHandleDimension",has_key);
433 _default_handle_dimension = key_value;
434 BESDEBUG(HDF5_NAME, prolog <<
"H5.DefaultHandleDimension: " << (_default_handle_dimension?
"true":
"false") << endl);
438 if(get_lrdcache_entries()) {
439 lrdata_mem_cache =
new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
440 bool has_LFMC_config =
false;
441 key_value = obtain_beskeys_info(
"H5.LargeDataMemCacheConfig",has_key);
443 has_LFMC_config = key_value;
444 BESDEBUG(HDF5_NAME, prolog <<
"H5.LargeDataMemCacheConfig: " << (has_LFMC_config?
"true":
"false") << endl);
445 if(
true == has_LFMC_config) {
446 _common_cache_dirs =obtain_lrd_common_cache_dirs();
448 if(
false == _common_cache_dirs)
449cerr<<
"No specific cache info"<<endl;
453 if(get_srdcache_entries()) {
455 BESDEBUG(HDF5_NAME, prolog <<
"Generate memory cache for smaller coordinate variables" << endl);
456 srdata_mem_cache =
new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
460 if(_disk_cache_comp_data ==
true && _use_disk_cache ==
true) {
461 if(_disk_cache_comp_threshold < 1.0) {
463 ss<< _disk_cache_comp_threshold;
464 string _comp_threshold_str(ss.str());
465 string invalid_comp_threshold =
"The Compression Threshold is the total size of the variable array";
466 invalid_comp_threshold+=
" divided by the storage size of compressed array. It should always be >1";
467 invalid_comp_threshold+=
" The current threhold set at h5.conf is ";
468 invalid_comp_threshold+=_comp_threshold_str;
469 invalid_comp_threshold+=
" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
473 _stp_east_filename = get_beskeys(
"H5.STPEastFileName");
474 _stp_north_filename = get_beskeys(
"H5.STPNorthFileName");
476 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
483 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
484#if DYNAMIC_CONFIG_ENABLED
489 hid_t cf_fileid = -1;
504 DAS *das = bdas->get_das();
507 DAS *cached_das_ptr =
nullptr;
508 bool use_das_cache =
false;
510 cached_das_ptr =
static_cast<DAS*
>(das_cache->
get(filename));
512 use_das_cache =
true;
514 if (
true == use_das_cache) {
517 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
518 *das = *cached_das_ptr;
522 bool das_from_dc =
false;
523 string das_cache_fname;
526 if(_use_disk_meta_cache ==
true) {
528 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
529 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
531 if(access(das_cache_fname.c_str(),F_OK) !=-1)
537 if(
true == das_from_dc) {
538 read_das_from_disk_cache(das_cache_fname,das);
543 BESDEBUG(HDF5_NAME, prolog <<
"HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
544 das_cache->
add(
new DAS(*das), filename);
549 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
550 if (
true == _usecf) {
552 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
554 string invalid_file_msg=
"Could not open this HDF5 file ";
555 invalid_file_msg +=filename;
556 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
557 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
558 invalid_file_msg +=
" distributor.";
562 if(HDF5RequestHandler::get_dmr_64bit_int()!=
nullptr)
563 HDF5RequestHandler::set_dmr_64bit_int(
nullptr);
564 read_cfdas( *das,filename,cf_fileid);
570 string invalid_file_msg=
"Could not open this HDF5 file ";
571 invalid_file_msg +=filename;
572 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
573 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
574 invalid_file_msg +=
" distributor.";
584 Ancillary::read_ancillary_das( *das, filename ) ;
588AttrTable* top_table = das->get_top_level_attributes();
589get_attr_contents(top_table);
592AttrTable::Attr_iter start_aiter=das->var_begin();
593AttrTable::Attr_iter it = start_aiter;
594AttrTable::Attr_iter end_aiter = das->var_end();
595while(it != end_aiter) {
596AttrTable* temp_table = das->get_table(it);
598cerr<<
"var_begin"<<endl;
599temp_table->print(cerr);
607 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
608 das_cache->
add(
new DAS(*das), filename);
613 if(das_cache_fname!=
"") {
614 BESDEBUG(HDF5_NAME, prolog <<
"HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
615 write_das_to_disk_cache(das_cache_fname,das);
625 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
628 catch(InternalErr & e) {
632 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
639 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
646 string s =
"unknown exception caught building HDF5 DAS";
650 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
657void HDF5RequestHandler::get_dds_with_attributes(
BESDDSResponse*bdds,
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename,
const string &dds_cache_fname,
const string &das_cache_fname,
bool dds_from_dc,
bool das_from_dc,
bool build_data)
660 if(
true == build_data)
661 dds = data_bdds->get_dds();
666 hid_t cf_fileid = -1;
671 DDS* cached_dds_ptr =
nullptr;
672 bool use_dds_cache =
false;
674 cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename));
676 use_dds_cache =
true;
677 if (
true == use_dds_cache) {
680 BESDEBUG(HDF5_NAME, prolog <<
"DDS Metadata Cached hit for : " << filename << endl);
681 *dds = *cached_dds_ptr;
683 else if (
true ==dds_from_dc) {
684 read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
687 BESDEBUG(HDF5_NAME, prolog <<
"Build DDS from the HDF5 file. " << filename << endl);
688 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
689 dds->filename(filename);
694 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
696 string invalid_file_msg=
"Could not open this HDF5 file ";
697 invalid_file_msg +=filename;
698 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
699 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
700 invalid_file_msg +=
" distributor.";
705 if(HDF5RequestHandler::get_dmr_64bit_int() !=
nullptr)
706 HDF5RequestHandler::set_dmr_64bit_int(
nullptr);
707 read_cfdds(*dds,filename,cf_fileid);
713 string invalid_file_msg=
"Could not open this HDF5 file ";
714 invalid_file_msg +=filename;
715 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
716 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
717 invalid_file_msg +=
" distributor.";
721 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
725 if (!dds->check_semantics()) {
727 throw InternalErr(__FILE__, __LINE__,
728 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
731 Ancillary::read_ancillary_dds( *dds, filename ) ;
734 if(dds_cache_fname!=
"" && dds_from_dc ==
false)
735 write_dds_to_disk_cache(dds_cache_fname,dds);
744 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
750 BESDEBUG(HDF5_NAME, prolog <<
"DDS added to the cache for : " << filename << endl);
751 dds_cache->
add(
new DDS(*dds), filename);
762 catch(InternalErr & e) {
770 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
780 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
790 string s =
"unknown exception caught building HDF5 DDS";
796void HDF5RequestHandler::get_dds_without_attributes_datadds(
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename)
798 DDS *dds = data_bdds->get_dds();
802 hid_t cf_fileid = -1;
807 DDS* cached_dds_ptr =
nullptr;
808 bool use_datadds_cache =
false;
810 cached_dds_ptr =
static_cast<DDS*
>(datadds_cache->
get(filename));
812 use_datadds_cache =
true;
813 if (
true == use_datadds_cache) {
816 BESDEBUG(HDF5_NAME, prolog <<
"DataDDS Metadata Cached hit for : " << filename << endl);
817 *dds = *cached_dds_ptr;
820 BESDEBUG(HDF5_NAME, prolog <<
"Build DDS from the HDF5 file. " << filename << endl);
821 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
822 dds->filename(filename);
827 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
829 string invalid_file_msg=
"Could not open this HDF5 file ";
830 invalid_file_msg +=filename;
831 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
832 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
833 invalid_file_msg +=
" distributor.";
838 if(HDF5RequestHandler::get_dmr_64bit_int() !=
nullptr)
839 HDF5RequestHandler::set_dmr_64bit_int(
nullptr);
840 read_cfdds(*dds,filename,cf_fileid);
846 string invalid_file_msg=
"Could not open this HDF5 file ";
847 invalid_file_msg +=filename;
848 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
849 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
850 invalid_file_msg +=
" distributor.";
854 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
858 if (!dds->check_semantics()) {
860 throw InternalErr(__FILE__, __LINE__,
861 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
864 Ancillary::read_ancillary_dds( *dds, filename ) ;
874 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
881 BESDEBUG(HDF5_NAME, prolog <<
"DataDDS added to the cache for : " << filename << endl);
882 datadds_cache->
add(
new DDS(*dds), filename);
891 BESDEBUG(HDF5_NAME, prolog <<
"Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
892 data_bdds->set_ia_flag(
false);
895 catch(InternalErr & e) {
903 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
913 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
923 string s =
"unknown exception caught building HDF5 DDS";
933void HDF5RequestHandler::get_dds_with_attributes(
const string &filename,
const string &container_name, DDS*dds) {
937 hid_t cf_fileid = -1;
942 DDS* cached_dds_ptr = 0;
943 if (dds_cache && (cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename)))) {
946 BESDEBUG(HDF5_NAME, prolog <<
"DDS Cached hit for : " << filename << endl);
947 *dds = *cached_dds_ptr;
952 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
953 if (!container_name.empty())
954 dds->container_name(container_name);
955 dds->filename(filename);
961 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
962 string dds_filename =
"/tmp/"+base_filename+
"_dds";
963 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
964cerr<<
"before parsing "<<endl;
966DDS tdds(&tf,name_path(filename),
"3.2");
967tdds.filename(filename);
969 tdds.parse(dds_file);
971 cache_dds =
new DDS(tdds);
977cerr<<
"after parsing "<<endl;
986 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
988 string invalid_file_msg=
"Could not open this HDF5 file ";
989 invalid_file_msg +=filename;
990 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
991 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
992 invalid_file_msg +=
" distributor.";
996 read_cfdds(*dds,filename,cf_fileid);
999 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1000 string dds_filename =
"/tmp/"+base_filename+
"_dds";
1001 FILE *dds_file = fopen(dds_filename.c_str(),
"w");
1002 dds->print(dds_file);
1010 string invalid_file_msg=
"Could not open this HDF5 file ";
1011 invalid_file_msg +=filename;
1012 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1013 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1014 invalid_file_msg +=
" distributor.";
1018 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
1024 if (!dds->check_semantics()) {
1026 throw InternalErr(__FILE__, __LINE__,
1027 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
1030 Ancillary::read_ancillary_dds( *dds, filename ) ;
1036 if (das_cache && (das =
static_cast<DAS*
>(das_cache->
get(filename)))) {
1037 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
1038 dds->transfer_attributes(das);
1045 if (!container_name.empty())
1046 das->container_name(container_name);
1048 if (
true == _usecf) {
1051 read_cfdas( *das,filename,cf_fileid);
1062 H5Fclose(cf_fileid);
1064 Ancillary::read_ancillary_das( *das, filename ) ;
1066 dds->transfer_attributes(das);
1072 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
1074 das_cache->
add(das, filename);
1083 BESDEBUG(HDF5_NAME, prolog <<
"DDS added to the cache for : " << filename << endl);
1084 dds_cache->
add(
new DDS(*dds), filename);
1092 catch(InternalErr & e) {
1095 H5Fclose(cf_fileid);
1100 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1101 __FILE__, __LINE__);
1106 H5Fclose(cf_fileid);
1110 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1111 __FILE__, __LINE__);
1116 H5Fclose(cf_fileid);
1120 string s =
"unknown exception caught building HDF5 DDS";
1130 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1131#if DYNAMIC_CONFIG_ENABLED
1147 bool dds_from_dc =
false;
1148 bool das_from_dc =
false;
1149 bool build_data =
false;
1150 string dds_cache_fname;
1151 string das_cache_fname;
1153 if(_use_disk_meta_cache) {
1155 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1158 if(_use_disk_dds_cache) {
1159 dds_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_dds";
1160 if(access(dds_cache_fname.c_str(),F_OK) !=-1)
1164 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
1166 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1171 get_dds_with_attributes(bdds,
nullptr,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1175 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1176 string dds_filename =
"/tmp/"+base_filename+
"_dds";
1179 DDS tdds(&tf,name_path(filename),
"3.2");
1180 tdds.filename(filename);
1183 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
1184 tdds.parse(dds_file);
1186 DDS* cache_dds =
new DDS(tdds);
1198 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1201 catch(InternalErr & e) {
1203 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1204 __FILE__, __LINE__);
1208 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1209 __FILE__, __LINE__);
1213 string s =
"unknown exception caught building HDF5 DDS";
1217 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1223 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1224#if DYNAMIC_CONFIG_ENABLED
1230 if(
true == _pass_fileid)
1231 return hdf5_build_data_with_IDs(dhi);
1248 bool dds_from_dc =
false;
1249 bool build_data =
true;
1252 bool das_from_dc =
false;
1253 string dds_cache_fname;
1254 string das_cache_fname;
1258 if(_use_disk_meta_cache ==
true) {
1260 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1261 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
1263 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1269 get_dds_with_attributes(
nullptr,bdds, container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1271 get_dds_without_attributes_datadds(bdds,container_name,filename);
1278 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1281 catch(InternalErr & e) {
1283 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1284 __FILE__, __LINE__);
1288 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1289 __FILE__, __LINE__);
1293 string s =
"unknown exception caught building HDF5 DDS";
1297 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1304 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1305#if DYNAMIC_CONFIG_ENABLED
1309 BESDEBUG(HDF5_NAME,prolog <<
"Building DataDDS by passing file IDs. "<<endl);
1310 hid_t cf_fileid = -1;
1314 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
1315 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1317 string invalid_file_msg=
"Could not open this HDF5 file ";
1318 invalid_file_msg +=filename;
1319 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1320 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1321 invalid_file_msg +=
" distributor.";
1334 auto hdds =
new HDF5DDS(bdds->get_dds());
1335 delete bdds->get_dds();
1339 hdds->setHDF5Dataset(cf_fileid);
1341 read_cfdds( *hdds,filename,cf_fileid);
1343 if (!hdds->check_semantics()) {
1345 throw InternalErr(__FILE__, __LINE__,
1346 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1349 Ancillary::read_ancillary_dds( *hdds, filename ) ;
1351 DAS *das =
new DAS ;
1354 read_cfdas( *das,filename,cf_fileid);
1355 Ancillary::read_ancillary_das( *das, filename ) ;
1357 hdds->transfer_attributes(das);
1365 H5Fclose(cf_fileid);
1366 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1369 catch(InternalErr & e) {
1371 H5Fclose(cf_fileid);
1372 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1373 __FILE__, __LINE__);
1377 H5Fclose(cf_fileid);
1378 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1379 __FILE__, __LINE__);
1383 H5Fclose(cf_fileid);
1384 string s =
"unknown exception caught building HDF5 DataDDS";
1388 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1394 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1395#if DYNAMIC_CONFIG_ENABLED
1406 DMR *dmr = bes_dmr_response.get_dmr();
1410 hid_t cf_fileid = -1;
1414 DMR* cached_dmr_ptr =
nullptr;
1416 BESDEBUG(HDF5_NAME, prolog <<
"Checking DMR cache for : " << filename << endl);
1417 cached_dmr_ptr =
static_cast<DMR*
>(dmr_cache->
get(filename));
1420 if (cached_dmr_ptr) {
1422 BESDEBUG(HDF5_NAME, prolog <<
"DMR cache hit for : " << filename << endl);
1423 *dmr = *cached_dmr_ptr;
1428 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
1429 D4BaseTypeFactory MyD4TypeFactory;
1430 dmr->set_factory(&MyD4TypeFactory);
1434 if(
true == _usecfdmr) {
1435 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1437 string invalid_file_msg=
"Could not open this HDF5 file ";
1438 invalid_file_msg +=filename;
1439 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1440 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1441 invalid_file_msg +=
" distributor.";
1444 read_cfdmr(dmr,filename,cf_fileid);
1445 H5Fclose(cf_fileid);
1448 dmr->set_factory(
nullptr);
1450 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1455 if(
true == _pass_fileid)
1456 return hdf5_build_dmr_with_IDs(dhi);
1458 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1460 string invalid_file_msg=
"Could not open this HDF5 file ";
1461 invalid_file_msg +=filename;
1462 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1463 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1464 invalid_file_msg +=
" distributor.";
1470 DDS dds(&factory, name_path(filename),
"3.2");
1471 dds.filename(filename);
1477 HDF5RequestHandler::set_dmr_64bit_int(dmr);
1478 read_cfdds( dds,filename,cf_fileid);
1479 if (!dds.check_semantics()) {
1481 throw InternalErr(__FILE__, __LINE__,
1482 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1485 read_cfdas(das,filename,cf_fileid);
1486 Ancillary::read_ancillary_das( das, filename ) ;
1488 dds.transfer_attributes(&das);
1492 H5Fclose(cf_fileid);
1494 dmr->build_using_dds(dds);
1502 string invalid_file_msg=
"Could not open this HDF5 file ";
1503 invalid_file_msg +=filename;
1504 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1505 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1506 invalid_file_msg +=
" distributor.";
1510 bool use_dimscale =
false;
1511 if(
true == _default_handle_dimension)
1512 use_dimscale = check_dimscale(fileid);
1513 dmr->set_name(name_path(filename));
1514 dmr->set_filename(name_path(filename));
1519 D4Group* root_grp = dmr->root();
1520 BESDEBUG(
"h5",
"use_dimscale is "<< use_dimscale <<endl);
1528 vector<link_info_t> hdf5_hls;
1529 breadth_first(fileid, fileid,(
char*)
"/",root_grp,filename.c_str(),use_dimscale,hdf5_hls);
1531 BESDEBUG(
"h5",
"build_dmr - before obtain dimensions"<< endl);
1532 D4Dimensions *root_dims = root_grp->dims();
1533 for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1534 BESDEBUG(
"fonc",
"transform_dap4() - check dimensions"<< endl);
1535 BESDEBUG(
"fonc",
"transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1536 BESDEBUG(
"fonc",
"transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1537 BESDEBUG(
"fonc",
"transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1541 BESDEBUG(
"h5",
"build_dmr - after obtain dimensions"<< endl);
1545 if(
true == use_dimscale)
1547 breadth_first(fileid,(
char*)
"/",root_grp,filename.c_str(),
true);
1549 depth_first(fileid,(
char*)
"/",root_grp,filename.c_str());
1560 BESDEBUG(HDF5_NAME, prolog <<
"DMR added to the cache for : " << filename << endl);
1561 dmr_cache->
add(
new DMR(*dmr), filename);
1567 H5Fclose(cf_fileid);
1570 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1573 catch(InternalErr & e) {
1576 H5Fclose(cf_fileid);
1580 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1581 __FILE__, __LINE__);
1586 H5Fclose(cf_fileid);
1589 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1590 __FILE__, __LINE__);
1595 H5Fclose(cf_fileid);
1598 string s =
"unknown exception caught building HDF5 DMR";
1613 dmr->set_factory(
nullptr);
1615 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1622 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1623#if DYNAMIC_CONFIG_ENABLED
1627 BESDEBUG(
"h5",
"Building DMR with passing file IDs. "<<endl);
1629 hid_t cf_fileid = -1;
1631 H5Eset_auto2(H5E_DEFAULT,
nullptr,
nullptr);
1632 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1634 string invalid_file_msg=
"Could not open this HDF5 file ";
1635 invalid_file_msg +=filename;
1636 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1637 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1638 invalid_file_msg +=
" distributor.";
1643 DDS dds(&factory, name_path(filename),
"3.2");
1644 dds.filename(filename);
1652 read_cfdds( dds,filename,cf_fileid);
1654 if (!dds.check_semantics()) {
1656 throw InternalErr(__FILE__, __LINE__,
1657 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1660 Ancillary::read_ancillary_dds( dds, filename ) ;
1663 read_cfdas(das,filename,cf_fileid);
1665 Ancillary::read_ancillary_das( das, filename ) ;
1667 dds.transfer_attributes(&das);
1678 H5Fclose(cf_fileid);
1679 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1682 catch(InternalErr & e) {
1685 H5Fclose(cf_fileid);
1687 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1688 __FILE__, __LINE__);
1693 H5Fclose(cf_fileid);
1695 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1696 __FILE__, __LINE__);
1701 H5Fclose(cf_fileid);
1703 string s =
"unknown exception caught building HDF5 DataDDS";
1717 DMR *dmr = bes_dmr.get_dmr();
1718 D4BaseTypeFactory MyD4TypeFactory;
1719 dmr->set_factory(&MyD4TypeFactory);
1720 dmr->build_using_dds(dds);
1722 auto hdf5_dmr =
new HDF5DMR(dmr);
1723 hdf5_dmr->setHDF5Dataset(cf_fileid);
1725 bes_dmr.set_dmr(hdf5_dmr);
1734 hdf5_dmr->set_factory(
nullptr);
1736 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1747 string add_info=
"Just for Test";
1749 map<string,string> attrs ;
1750 attrs[
"name"] = MODULE_NAME ;
1751 attrs[
"version"] = MODULE_VERSION ;
1752 list<string> services ;
1753 BESServiceRegistry::TheRegistry()->
services_handled( HDF5_NAME, services );
1754 if( services.empty()==
false )
1757 attrs[
"handles"] = handles ;
1759 info->begin_tag(
"module", &attrs ) ;
1760 info->end_tag(
"module" ) ;
1773 info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1779bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1781 string lrd_config_fpath;
1782 string lrd_config_fname;
1785 lrd_config_fpath = get_beskeys(
"H5.DataCachePath");
1788 lrd_config_fname = get_beskeys(
"H5.LargeDataMemCacheFileName");
1791 if(lrd_config_fpath==
"" || lrd_config_fname==
"")
1798 string mcache_config_fname = lrd_config_fpath+
"/"+lrd_config_fname;
1804 ifstream mcache_config_file(mcache_config_fname.c_str());
1807 if(mcache_config_file.is_open()==
false){
1808 BESDEBUG(HDF5_NAME, prolog <<
"The large data memory cache configure file "<<mcache_config_fname );
1809 BESDEBUG(HDF5_NAME, prolog <<
" cannot be opened."<<endl);
1814 while(getline(mcache_config_file,temp_line)) {
1817 if(temp_line.size()>1 && temp_line.at(1)==
' ') {
1819 string subline = temp_line.substr(2);
1820 vector<string> temp_name_list;
1823 if(temp_line.at(0)==
'1') {
1824 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1828 lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1831 else if(temp_line.at(0)==
'0'){
1832 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1836 lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1839 else if(temp_line.at(0)==
'2') {
1845 for(
unsigned int i = 0; i<subline.size();i++){
1846 if(subline[i]==
'"') {
1847 dq_pos.push_back(i);
1849 else if(subline[i]==
'\'')
1850 sq_pos.push_back(i);
1852 if(dq_pos.empty() && sq_pos.empty())
1853 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1854 else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1855 unsigned int dq_index= 0;
1856 while(dq_index < dq_pos.size()){
1857 if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1858 temp_name_list.push_back
1859 (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1861 dq_index = dq_index + 2;
1864 else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1865 unsigned int sq_index= 0;
1866 while(sq_index < sq_pos.size()){
1867 if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1868 temp_name_list.push_back
1869 (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1871 sq_index = sq_index+2;
1875 lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1883for(
int i =0; i<lrd_cache_dir_list.size();i++)
1884cerr<<
"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1885for(
int i =0; i<lrd_non_cache_dir_list.size();i++)
1886cerr<<
"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1887for(
int i =0; i<lrd_var_cache_file_list.size();i++)
1888cerr<<
"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1892 mcache_config_file.close();
1893 if(lrd_cache_dir_list.empty() && lrd_non_cache_dir_list.empty() && lrd_var_cache_file_list.empty())
1900bool HDF5RequestHandler::read_das_from_disk_cache(
const string & cache_filename,DAS *das_ptr) {
1902 BESDEBUG(HDF5_NAME, prolog <<
"Coming to read_das_from_disk_cache() " << cache_filename << endl);
1903 bool ret_value =
true;
1904 FILE *md_file =
nullptr;
1905 md_file = fopen(cache_filename.c_str(),
"rb");
1907 if(
nullptr == md_file) {
1908 string bes_error =
"An error occurred trying to open a metadata cache file " + cache_filename;
1913 int fd_md = fileno(md_file);
1915 l_md = lock(F_RDLCK);
1918 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1921 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1928 if(stat(cache_filename.c_str(),&sb) != 0) {
1929 string bes_error =
"An error occurred trying to stat a metadata cache file size " + cache_filename;
1935 auto bytes_expected_read=(size_t)sb.st_size;
1936 BESDEBUG(HDF5_NAME, prolog <<
"DAS Disk cache file size is " << bytes_expected_read << endl);
1939 buf.resize(bytes_expected_read);
1940 size_t bytes_to_read =fread((
void*)buf.data(),1,bytes_expected_read,md_file);
1941 if(bytes_to_read != bytes_expected_read)
1942 throw InternalErr(__FILE__,__LINE__,
"Fail to read the data from the das cache file.");
1944 char* temp_pointer =buf.data();
1946 AttrTable*at =
nullptr;
1949 temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1955 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1957 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1961 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a das cache file.");
1965 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1967 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1976bool HDF5RequestHandler::write_dds_to_disk_cache(
const string& dds_cache_fname,DDS *dds_ptr) {
1978 BESDEBUG(HDF5_NAME, prolog <<
"Write DDS to disk cache " << dds_cache_fname << endl);
1979 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"w");
1981 if(
nullptr == dds_file) {
1982 string bes_error =
"An error occurred trying to open a metadata cache file " + dds_cache_fname;
1987 int fd_md = fileno(dds_file);
1989 l_md = lock(F_WRLCK);
1992 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1995 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
2000 dds_ptr->print(dds_file);
2003 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2005 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2009 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
2012 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2014 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2024bool HDF5RequestHandler::write_das_to_disk_cache(
const string & das_cache_fname, DAS *das_ptr) {
2026 BESDEBUG(HDF5_NAME, prolog <<
"Write DAS to disk cache " << das_cache_fname << endl);
2027 FILE *das_file = fopen(das_cache_fname.c_str(),
"wb");
2028 if(
nullptr == das_file) {
2029 string bes_error =
"An error occurred trying to open a metadata cache file " + das_cache_fname;
2033 int fd_md = fileno(das_file);
2035 l_md = lock(F_WRLCK);
2038 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
2041 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
2046 write_das_to_file(das_ptr,das_file);
2049 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2051 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2055 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
2058 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2060 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2072void write_das_to_file(DAS*das_ptr,FILE* das_file) {
2075 uint8_t category_flag = 2;
2076 AttrTable* top_table = das_ptr->get_top_level_attributes();
2077 write_das_table_to_file(top_table,das_file);
2080 fwrite((
const void*)&category_flag,1,1,das_file);
2086void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
2088 if(temp_table !=
nullptr) {
2091 uint8_t category_flag = 2;
2094 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2095 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2096 AttrTable::Attr_iter top_it = top_startit;
2097 while(top_it !=top_endit) {
2098 AttrType atype = temp_table->get_attr_type(top_it);
2099 if(atype == Attr_unknown)
2100 throw InternalErr(__FILE__,__LINE__,
"Unsupported DAS Attribute type");
2101 else if(atype!=Attr_container) {
2102 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr name is: "
2103 << temp_table->get_name(top_it) << endl);
2104 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr type is: "
2105 << temp_table->get_type(top_it) << endl);
2116 write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
2119 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr container name is: "
2120 << (*top_it)->name << endl);
2122 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2123 write_container_name_to_file(sub_table->get_name(),das_file);
2124 write_das_table_to_file(sub_table,das_file);
2127 fwrite((
const void*)&category_flag,1,1,das_file);
2137void write_container_name_to_file(
const string& cont_name,FILE *das_file) {
2140 uint8_t category_flag = 1;
2142 size_t bytes_to_write = cont_name.size()+
sizeof(size_t)+1;
2143 buf.resize(bytes_to_write);
2144 char*temp_pointer =buf.data();
2145 memcpy((
void*)temp_pointer,(
void*)&category_flag,1);
2147 temp_pointer=copy_str(temp_pointer,cont_name);
2149 size_t bytes_to_be_written = fwrite((
const void*)buf.data(),1,bytes_to_write,das_file);
2150 if(bytes_to_be_written != bytes_to_write)
2151 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS container name to a cache");
2157void write_das_attr_info(AttrTable* dtp,
const string& attr_name,
const string & attr_type,FILE * das_file) {
2160 uint8_t category_flag = 0;
2162 unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
2163 vector<string> attr_values;
2164 size_t total_attr_values_size = 0;
2165 for (
unsigned int i = 0; i <num_attr_elems;i++){
2166 attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
2167 total_attr_values_size += attr_values[i].size();
2171 size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2*
sizeof(size_t);
2177 bytes_to_write_attr +=
sizeof(
unsigned int) + num_attr_elems*
sizeof(
size_t)+total_attr_values_size;
2179 vector<char>attr_buf;
2180 attr_buf.resize(bytes_to_write_attr);
2181 char* temp_attrp =attr_buf.data();
2184 memcpy((
void*)temp_attrp,(
void*)&category_flag,1);
2188 temp_attrp=copy_str(temp_attrp,attr_name);
2189 temp_attrp=copy_str(temp_attrp,attr_type);
2192 memcpy((
void*)temp_attrp,(
void*)&num_attr_elems,
sizeof(
unsigned int));
2193 temp_attrp+=
sizeof(
unsigned int);
2196 for (
unsigned int i = 0; i <num_attr_elems;i++)
2197 temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
2199 size_t bytes_to_be_written = fwrite((
const void*)attr_buf.data(),1,bytes_to_write_attr,das_file);
2200 if(bytes_to_be_written != bytes_to_write_attr)
2201 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS attribute to a cache");
2209 bool build_data,
const string & container_name,
const string & h5_fname,
2210 const string & dds_cache_fname,
const string &das_cache_fname, hid_t h5_fd,
2214 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN dds_cache_fname: " << dds_cache_fname << endl);
2217 if(
true == build_data)
2218 dds = data_bdds->get_dds();
2224 DDS tdds(&tf,name_path(h5_fname),
"3.2");
2225 tdds.filename(h5_fname);
2227 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"r");
2228 tdds.parse(dds_file);
2229 auto cache_dds =
new DDS(tdds);
2231cerr<<
"before dds "<<endl;
2233cerr<<
"after dds "<<endl;
2234cerr<<
"before tdds "<<endl;
2235cache_dds->dump(cerr);
2236cerr<<
"after tdds "<<endl;
2241 Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
2243 add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
2244 if(
true == build_data)
2245 data_bdds->
set_dds(cache_dds);
2248 if(dds_file !=
nullptr)
2253 BESDEBUG(HDF5_NAME, prolog <<
"For memory cache, DDS added to the cache for : " << h5_fname << endl);
2254 dds_cache->
add(
new DDS(*cache_dds), h5_fname);
2260void HDF5RequestHandler::add_das_to_dds(DDS *dds,
const string &,
const string &filename,
2261 const string &das_cache_fname, hid_t h5_fd,
bool das_from_dc) {
2263 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
2266 DAS *das = nullptr ;
2267 bool use_das_cache =
false;
2269 das =
static_cast<DAS*
>(das_cache->
get(filename));
2271 use_das_cache =
true;
2273 if (
true == use_das_cache) {
2274 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
2275 dds->transfer_attributes(das);
2288 if (!container_name.empty())
2289 das->container_name(container_name);
2291 if(das_from_dc ==
true)
2292 read_das_from_disk_cache(das_cache_fname,das);
2295 bool h5_file_open =
true;
2297 h5_file_open =
false;
2298 if (
true == _usecf) {
2300 if(h5_file_open ==
false)
2301 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2303 read_cfdas( *das,filename,h5_fd);
2304 if(h5_file_open ==
false)
2308 if(h5_file_open ==
false)
2312 if(h5_file_open ==
false)
2316 Ancillary::read_ancillary_das( *das, filename ) ;
2318 if(das_cache_fname!=
"" && das_from_dc ==
false)
2319 write_das_to_disk_cache(das_cache_fname,das);
2322 dds->transfer_attributes(das);
2326 BESDEBUG(HDF5_NAME, prolog <<
"For memory cache, DAS added to the cache for : " << filename << endl);
2327 das_cache->
add(
new DAS(*das), filename);
2335bool obtain_beskeys_info(
const string& key,
bool & has_key) {
2337 bool ret_value =
false;
2341 const string dosettrue =
"true";
2342 const string dosetyes =
"yes";
2344 ret_value = (dosettrue == doset || dosetyes == doset);
2350bool is_beskeys_set_true(
const string& bes_value) {
2352 const string dosettrue =
"true";
2353 const string dosetyes =
"yes";
2355 return (dosettrue == doset || dosetyes == doset);
2358bool check_and_set_beskeys(
const string key) {
2362 const string dosettrue =
"true";
2363 const string dosetyes =
"yes";
2366 if(
true == found ) {
2369 BESDEBUG(HDF5_NAME, prolog <<
"Key: " << key << (found?(
" was found. value: "+doset):
" was not found.") << endl);
2370 return found && (dosettrue == doset || dosetyes == doset);
2377static unsigned int get_uint_key(
const string &key,
unsigned int def_val)
2383 if (
true == found) {
2385 return atoi(doset.c_str());
2392static unsigned long get_ulong_key(
const string &key,
unsigned long def_val)
2398 if (
true == found) {
2400 return atol(doset.c_str());
2406static float get_float_key(
const string &key,
float def_val)
2412 if (
true == found) {
2413 return atof(doset.c_str());
2420static string get_beskeys(
const string &key) {
2423 string ret_value =
"";
2431char* copy_str(
char*temp_ptr,
const string & str) {
2433 size_t str_size=str.size();
2434 memcpy((
void*)temp_ptr,(
void*)&str_size,
sizeof(
size_t));
2435 temp_ptr+=
sizeof(size_t);
2436 vector<char>temp_vc2(str.begin(),str.end());
2437 memcpy((
void*)temp_ptr,(
void*)temp_vc2.data(),str.size());
2438 temp_ptr+=str.size();
2447char* obtain_str(
char*temp_ptr,
string & str) {
2449 size_t oname_size = *((
size_t *)temp_ptr);
2450 temp_ptr = temp_ptr +
sizeof(size_t);
2452 for(
unsigned int i =0; i<oname_size; i++){
2453 oname.push_back(*temp_ptr);
2464char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at_par) {
2469 flag = *((uint8_t*)(temp_pointer));
2470 BESDEBUG(HDF5_NAME, prolog <<
"Build DAS from the disk cache file flag: "
2471 <<
" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2472 <<
" flag = 3; the initial value to get the attribute retrieval process started."
2473 <<
" The flag value is "
2474 << (
int)flag <<endl);
2478 string container_name;
2479 temp_pointer = obtain_str(temp_pointer,container_name);
2480 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, container name is " << container_name << endl);
2483 AttrTable*temp_at_par = at_par;
2484 if(at_par ==
nullptr)
2485 at_par = das->add_table(container_name,
new AttrTable);
2487 at_par = at_par->append_container(container_name);
2489 temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2491 at_par = temp_at_par;
2494 else if(flag == 0) {
2496 if(at_par ==
nullptr)
2497 throw BESInternalError(
"The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2501 temp_pointer = obtain_str(temp_pointer,attr_name);
2502 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attr name is: " << attr_name << endl);
2506 temp_pointer = obtain_str(temp_pointer,attr_type);
2507 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attr type is: " << attr_type << endl);
2510 unsigned int num_values = *((
unsigned int*)(temp_pointer));
2511 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, number of attribute values is: " << num_values << endl);
2512 temp_pointer+=
sizeof(
unsigned int);
2514 vector <string> attr_values;
2516 for(
unsigned int i = 0; i<num_values; i++) {
2518 temp_pointer = obtain_str(temp_pointer,attr_value);
2519 attr_values.push_back(attr_value);
2520 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attribute value is: " << attr_value << endl);
2523 at_par->append_attr(attr_name,attr_type,&attr_values);
2527 return temp_pointer;
2532void get_attr_contents(AttrTable*temp_table) {
2533 if(temp_table !=
nullptr) {
2534 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2535 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2536 AttrTable::Attr_iter top_it = top_startit;
2537 while(top_it !=top_endit) {
2538 AttrType atype = temp_table->get_attr_type(top_it);
2539 if(atype == Attr_unknown)
2540 cerr<<
"unsupported DAS attributes" <<endl;
2541 else if(atype!=Attr_container) {
2543 cerr<<
"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2544 cerr<<
"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2545 unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2546 cerr<<
"Attribute values are "<<endl;
2547 for (
unsigned int i = 0; i <num_attrs;i++)
2548 cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<
" ";
2552 cerr<<
"Coming to the attribute container. "<<endl;
2553 cerr<<
"container name is "<<(*top_it)->name <<endl;
2554 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2555 cerr<<
"container table name is "<<sub_table->get_name() <<endl;
2556 get_attr_contents(sub_table);
2572 DDS *dds = bdds->get_dds();
2576 bool das_from_mcache =
false;
2578 das =
static_cast<DAS*
>(das_cache->
get(filename));
2580 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
2581 dds->transfer_attributes(das);
2582 das_from_mcache =
true;
2586 if(
false == das_from_mcache) {
2590 if (!container_name.empty()) das->container_name(container_name);
2593 if (
true == _usecf) {
2595 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2597 read_cfdas( *das,filename,h5_fd);
2609 Ancillary::read_ancillary_das(*das, filename);
2611 dds->transfer_attributes(das);
2616 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
2617 das_cache->
add(das, filename);
2623 BESDEBUG(HDF5_NAME, prolog <<
"Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2624 bdds->set_ia_flag(
true);
include the entry functions to execute the handlers
std::string get_symbolic_name() const
retrieve the symbolic name for this container
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
BESContainer * container
pointer to current container in this interface
Base exception class for the BES with basic string message.
std::string get_message() const
get the error message for this exception
informational response object
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
static std::string lowercase(const std::string &s)
static std::string implode(const std::list< std::string > &values, char delim)
An in-memory cache for DapObj (DAS, DDS, ...) objects.
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
Helper functions for generating DAS attributes and a function to check BES Key.
void depth_first(hid_t pid, const char *gname, DAS &das)
void find_gloattr(hid_t file, DAS &das)
bool breadth_first(const hid_t file_id, hid_t pid, const char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
hid_t get_fileid(const char *filename)
void close_fileid(hid_t fid)
The main header of the HDF5 OPeNDAP handler.