bes Updated for version 3.20.13
HDF5RequestHandler.cc
Go to the documentation of this file.
1
2// -*- mode: c++; c-basic-offset:4 -*-
3
4// This file is part of hdf5_handler, a data handler for the OPeNDAP data
5// server.
6
7// Copyright (c) 2002,2003 OPeNDAP, Inc.
8// Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
9// Author: James Gallagher <jgallagher@opendap.org>
10//
11// This is free software; you can redistribute it and/or modify it under the
12// terms of the GNU Lesser General Public License as published by the Free
13// Software Foundation; either version 2.1 of the License, or (at your
14// option) any later version.
15//
16// This software is distributed in the hope that it will be useful, but
17// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
19// License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26
27// HDF5RequestHandler.cc
33
34
35#include <iostream>
36#include <fstream>
37#include <sstream>
38#include <string>
39#include <sys/types.h>
40#include <sys/stat.h>
41#include <fcntl.h>
42#include <unistd.h>
43
44#include <libdap/DMR.h>
45#include <libdap/D4BaseTypeFactory.h>
46#include <BESDMRResponse.h>
47#include <ObjMemCache.h>
48#include "HDF5_DMR.h"
49
50#include <libdap/mime_util.h>
51#include "hdf5_handler.h"
52#include "HDF5RequestHandler.h"
53#include "HDF5_DDS.h"
54
55#include <BESDASResponse.h>
56#include <BESDDSResponse.h>
57#include <BESDataDDSResponse.h>
58#include <libdap/Ancillary.h>
59#include <BESInfo.h>
60#include <BESDapNames.h>
61#include <BESResponseNames.h>
62#include <BESContainer.h>
63#include <BESResponseHandler.h>
64#include <BESVersionInfo.h>
65#include <BESServiceRegistry.h>
66#include <BESUtil.h>
67#include <BESDapError.h>
68#include <BESInternalFatalError.h>
69#include <TheBESKeys.h>
70#include <BESDebug.h>
71#include <BESStopWatch.h>
72#include "h5get.h"
73#include "config_hdf5.h"
74
75#define HDF5_NAME "h5"
76#include "h5cfdaputil.h"
77
78using namespace std;
79using namespace libdap;
80
81// DYNAMIC_CONFIG_ENABLED is defined by TheBESKeys.h. As of 3/11/22, this
82// is 0 so the dynamic keys feature is not used. jhrg 3/8/22
83
84#define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
85
86// The debug function to dump all the contents of a DAS table.
87void get_attr_contents(AttrTable* temp_table);
88
89// Five functions to generate a DAS cache file.
90// 1. The wrapper function to call write_das_table_to_file to generate the cache.
91void write_das_to_file(DAS*das_ptr,FILE* das_file);
92
93// 2. The main function to generate the DAS cache.
94void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
95
96// 3. The function to write the DAS container/table name to the cache
97void write_container_name_to_file(const string&,FILE* das_file);
98
99// 4. The function to write the DAS attribute name,type and values to the cache
100void write_das_attr_info(AttrTable* dtp,const string&,const string&,FILE * das_file);
101
102// 5. The function to copy a string to a memory buffer.
103char* copy_str(char *temp_ptr,const string & str);
104
105// These two functions are for reading the DAS from the DAS cache file
106// 1. Obtain the string from a memory buffer.
107char* obtain_str(char*temp_ptr,string & str);
108
109// 2. The main function to obtain DAS info from the cache.
110char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at);
111
112#if 0
113// Check if the BES key is set.
114//bool check_and_set_beskeys(const string);
115#endif
116
117// Obtain the BES key as an integer
118static unsigned int get_uint_key(const string &key,unsigned int def_val);
119static unsigned long get_ulong_key(const string &key,unsigned long def_val);
120
121// Obtain the BES key as a floating-pointer number.
122static float get_float_key(const string &key, float def_val);
123
124// Obtain the BES key as a string.
125static string get_beskeys(const string&);
126
127#if 0
128//static bool is_beskeys_set_true(const string &);
129#endif
130
131bool obtain_beskeys_info(const string &, bool &);
132
133// For the CF option
134extern void read_cfdas(DAS &das, const string & filename,hid_t fileid);
135extern void read_cfdds(DDS &dds, const string & filename,hid_t fileid);
136extern void read_cfdmr(DMR *dmr, const string & filename,hid_t fileid);
137
138
139
140// Check the description of cache_entries and cache_purge_level at h5.conf.in.
141unsigned int HDF5RequestHandler::_mdcache_entries = 500;
142unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
143unsigned int HDF5RequestHandler::_srdcache_entries = 0;
144float HDF5RequestHandler::_cache_purge_level = 0.2F;
145
146// Metadata object cache at DAS,DDS and DMR.
147ObjMemCache *HDF5RequestHandler::das_cache = nullptr;
148ObjMemCache *HDF5RequestHandler::dds_cache = nullptr;
149ObjMemCache *HDF5RequestHandler::datadds_cache = nullptr;
150ObjMemCache *HDF5RequestHandler::dmr_cache = nullptr;
151
152ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = nullptr;
153ObjMemCache *HDF5RequestHandler::srdata_mem_cache = nullptr;
154
155// Set default values of all BES keys according to h5.conf.in.
156// This will help the generation of DMRPP. No need to
157// set multiple keys if the user's setting is the same as
158// the h5.conf.in. KY 2021-08-23
159bool HDF5RequestHandler::_usecf = true;
160bool HDF5RequestHandler::_pass_fileid = false;
161bool HDF5RequestHandler::_disable_structmeta = true;
162bool HDF5RequestHandler::_disable_ecsmeta = false;
163bool HDF5RequestHandler::_keep_var_leading_underscore = false;
164bool HDF5RequestHandler::_check_name_clashing = false;
165bool HDF5RequestHandler::_add_path_attrs = true;
166bool HDF5RequestHandler::_drop_long_string = true;
167bool HDF5RequestHandler::_fillvalue_check = true;
168bool HDF5RequestHandler::_check_ignore_obj = false;
169bool HDF5RequestHandler::_flatten_coor_attr = true;
170bool HDF5RequestHandler::_default_handle_dimension = true; //Ignored when _usecf=true.
171bool HDF5RequestHandler::_eos5_rm_convention_attr_path = true;
172bool HDF5RequestHandler::_dmr_long_int = true;
173bool HDF5RequestHandler::_no_zero_size_fullnameattr = false;
174bool HDF5RequestHandler::_enable_coord_attr_add_path = true;
175
176bool HDF5RequestHandler::_usecfdmr = true;
177bool HDF5RequestHandler::_add_dap4_coverage = true;
178
179bool HDF5RequestHandler::_common_cache_dirs = false;
180
181bool HDF5RequestHandler::_use_disk_cache =false;
182bool HDF5RequestHandler::_use_disk_dds_cache =false;
183string HDF5RequestHandler::_disk_cache_dir ="";
184string HDF5RequestHandler::_disk_cachefile_prefix ="";
185unsigned long long HDF5RequestHandler::_disk_cache_size =0;
186
187
188bool HDF5RequestHandler::_disk_cache_comp_data =false;
189bool HDF5RequestHandler::_disk_cache_float_only_comp_data =false;
190float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
191unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
192
193bool HDF5RequestHandler::_use_disk_meta_cache = false;
194string HDF5RequestHandler::_disk_meta_cache_path ="";
195
196bool HDF5RequestHandler::_use_latlon_disk_cache = false;
197long HDF5RequestHandler::_latlon_disk_cache_size =0;
198string HDF5RequestHandler::_latlon_disk_cache_dir ="";
199string HDF5RequestHandler::_latlon_disk_cachefile_prefix="";
200
201// H5.EscapeUTF8Attr jhrg 3/9/22
202bool HDF5RequestHandler::_escape_utf8_attr = true;
203
204DMR* HDF5RequestHandler::dmr_int64 = nullptr;
205
206
207#if 0
208//BaseTypeFactory factory;
209//libdap::DDS HDF5RequestHandler::hd_dds(&factory,"");
210#endif
211string HDF5RequestHandler::_stp_east_filename;
212string HDF5RequestHandler::_stp_north_filename;
213vector<string> HDF5RequestHandler::lrd_cache_dir_list;
214vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
215vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
216
217#if 0
218//libdap::DDS*cache_dds;
219#endif
220
221HDF5RequestHandler::HDF5RequestHandler(const string & name)
222 :BESRequestHandler(name)
223{
224
225 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
226
227 add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
228 add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
229 add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
230 add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
231 add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
232
233 add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
234 add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
235
236#if !(DYNAMIC_CONFIG_ENABLED)
237 load_config();
238#endif
239
240 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
241}
242
243HDF5RequestHandler::~HDF5RequestHandler()
244{
245 // delete the cache.
246 delete das_cache;
247 delete dds_cache;
248 delete datadds_cache;
249 delete dmr_cache;
250 delete lrdata_mem_cache;
251 delete srdata_mem_cache;
252
253}
254
258void HDF5RequestHandler::load_config()
259{
260 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
261 BESStopWatch sw;
262 if (BESISDEBUG(HDF5_NAME)){
263 sw.start(prolog,"ClockTheBESKeys");
264 }
265
266 // Obtain the metadata cache entries and purge level.
267 HDF5RequestHandler::_mdcache_entries = get_uint_key("H5.MetaDataMemCacheEntries", 0);
268 HDF5RequestHandler::_lrdcache_entries = get_uint_key("H5.LargeDataMemCacheEntries", 0);
269 HDF5RequestHandler::_srdcache_entries = get_uint_key("H5.SmallDataMemCacheEntries", 0);
270 HDF5RequestHandler::_cache_purge_level = get_float_key("H5.CachePurgeLevel", 0.2F);
271
272 if (get_mdcache_entries()) { // else it stays at its default of null
273 das_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
274 dds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
275 datadds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
276 dmr_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
277 }
278
279 // Starting from hyrax 1.16.5, users don't need to explicitly set the BES keys if
280 // they are happy with the default settings in the h5.conf.in.
281 // In the previous releases, we required users to set BES key values. Otherwise,
282 // the BES key values of true/false will always be false if the key is not found.
283 // Hopefully this change will make it convenient for the general users.
284 // In the mean time, the explicit BES key settings will not be affected.
285 // KY 2021-10-13
286 //
287 // Check if the EnableCF key is set.
288 bool has_key = false;
289 bool key_value = obtain_beskeys_info("H5.EnableCF",has_key);
290 if(has_key)
291 _usecf = key_value;
292 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCF: " << (_usecf?"true":"false") << endl);
293
294 // The DefaultHandleDimension is effective only when EnableCF=false
295 key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
296 if(has_key)
297 _default_handle_dimension = key_value;
298 BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
299
300 // The following keys are only effective when EnableCF is true or unset(EnableCF is true if users don't set the key).
301 key_value = obtain_beskeys_info("H5.EnablePassFileID",has_key);
302 if(has_key)
303 _pass_fileid = key_value;
304 BESDEBUG(HDF5_NAME, prolog << "H5.EnablePassFileID: " << (_pass_fileid?"true":"false") << endl);
305
306 key_value = obtain_beskeys_info("H5.DisableStructMetaAttr",has_key);
307 if(has_key)
308 _disable_structmeta = key_value;
309 BESDEBUG(HDF5_NAME, prolog << "H5.DisableStructMetaAttr: " << (_disable_structmeta?"true":"false") << endl);
310
311 key_value = obtain_beskeys_info("H5.DisableECSMetaAttr",has_key);
312 if(has_key)
313 _disable_ecsmeta = key_value;
314 BESDEBUG(HDF5_NAME, prolog << "H5.DisableECSMetaAttr: " << (_disable_ecsmeta?"true":"false") << endl);
315
316 key_value = obtain_beskeys_info("H5.KeepVarLeadingUnderscore",has_key);
317 if(has_key)
318 _keep_var_leading_underscore = key_value;
319 BESDEBUG(HDF5_NAME, prolog << "H5.KeepVarLeadingUnderscore: " << (_keep_var_leading_underscore?"true":"false") << endl);
320
321#if 0
322 //_check_name_clashing = check_and_set_beskeys("H5.EnableCheckNameClashing");
323#endif
324 key_value = obtain_beskeys_info("H5.EnableCheckNameClashing",has_key);
325 if(has_key)
326 _check_name_clashing = key_value;
327 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCheckNameClashing: " << (_check_name_clashing?"true":"false") << endl);
328
329 key_value = obtain_beskeys_info("H5.EnableAddPathAttrs",has_key);
330 if(has_key)
331 _add_path_attrs = key_value;
332 BESDEBUG(HDF5_NAME, prolog << "H5.EnableAddPathAttrs: " << (_add_path_attrs?"true":"false") << endl);
333
334 key_value = obtain_beskeys_info("H5.EnableDropLongString",has_key);
335 if(has_key)
336 _drop_long_string = key_value;
337 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDropLongString: " << (_drop_long_string?"true":"false") << endl);
338
339 key_value = obtain_beskeys_info("H5.EnableFillValueCheck",has_key);
340 if(has_key)
341 _fillvalue_check = key_value;
342 BESDEBUG(HDF5_NAME, prolog << "H5.EnableFillValueCheck: " << (_fillvalue_check?"true":"false") << endl);
343
344
345 key_value = obtain_beskeys_info("H5.CheckIgnoreObj",has_key);
346 if(has_key)
347 _check_ignore_obj = key_value;
348 BESDEBUG(HDF5_NAME, prolog << "H5.CheckIgnoreObj: " << (_check_ignore_obj?"true":"false") << endl);
349
350 key_value = obtain_beskeys_info("H5.ForceFlattenNDCoorAttr",has_key);
351 if(has_key)
352 _flatten_coor_attr = key_value;
353 BESDEBUG(HDF5_NAME, prolog << "H5.ForceFlattenNDCoorAttr: " << (_flatten_coor_attr?"true":"false") << endl);
354
355
356 key_value = obtain_beskeys_info("H5.RmConventionAttrPath",has_key);
357 if(has_key)
358 _eos5_rm_convention_attr_path = key_value;
359 BESDEBUG(HDF5_NAME, prolog << "H5.RmConventionAttrPath: " << (_eos5_rm_convention_attr_path?"true":"false") << endl);
360
361 key_value = obtain_beskeys_info("H5.EnableDMR64bitInt",has_key);
362 if(has_key)
363 _dmr_long_int = key_value;
364 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDMR64bitInt: " << (_dmr_long_int?"true":"false") << endl);
365
366 key_value = obtain_beskeys_info("H5.NoZeroSizeFullnameAttr",has_key);
367 if(has_key)
368 _no_zero_size_fullnameattr = key_value;
369 BESDEBUG(HDF5_NAME, prolog << "H5.NoZeroSizeFullnameAttr: " << (_no_zero_size_fullnameattr?"true":"false") << endl);
370
371 key_value = obtain_beskeys_info("H5.EnableCoorattrAddPath",has_key);
372 if(has_key)
373 _enable_coord_attr_add_path = key_value;
374 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCoorattrAddPath: " << (_enable_coord_attr_add_path?"true":"false") << endl);
375
376 key_value = obtain_beskeys_info("H5.EnableCFDMR",has_key);
377 if(has_key)
378 _usecfdmr = key_value;
379 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCFDMR: " << (_usecfdmr?"true":"false") << endl);
380
381 key_value = obtain_beskeys_info("H5.EnableDAP4Coverage",has_key);
382 if(has_key)
383 _add_dap4_coverage = key_value;
384 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDAP4Coverage: " << (_add_dap4_coverage?"true":"false") << endl);
385
386 key_value = obtain_beskeys_info("H5.EnableDiskDataCache",has_key);
387 if(has_key)
388 _use_disk_cache = key_value;
389 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDataCache: " << (_use_disk_cache?"true":"false") << endl);
390 _disk_cache_dir = get_beskeys("H5.DiskCacheDataPath");
391 _disk_cachefile_prefix = get_beskeys("H5.DiskCacheFilePrefix");
392 _disk_cache_size = get_ulong_key("H5.DiskCacheSize",0);
393
394 key_value = obtain_beskeys_info("H5.DiskCacheComp",has_key);
395 if(has_key)
396 _disk_cache_comp_data = key_value;
397 BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheComp: " << (_disk_cache_comp_data?"true":"false") << endl);
398
399 key_value = obtain_beskeys_info("H5.DiskCacheFloatOnlyComp",has_key);
400 if(has_key)
401 _disk_cache_float_only_comp_data = key_value;
402 BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheFloatOnlyComp: " << (_disk_cache_float_only_comp_data?"true":"false") << endl);
403 _disk_cache_comp_threshold = get_float_key("H5.DiskCacheCompThreshold",1.0);
404 _disk_cache_var_size = 1024*get_uint_key("H5.DiskCacheCompVarSize",0);
405
406 key_value = obtain_beskeys_info("H5.EnableDiskMetaDataCache",has_key);
407 if(has_key)
408 _use_disk_meta_cache = key_value;
409 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskMetaDataCache: " << (_use_disk_meta_cache?"true":"false") << endl);
410
411 key_value = obtain_beskeys_info("H5.EnableDiskDDSCache",has_key);
412 if(has_key)
413 _use_disk_dds_cache = key_value;
414 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDDSCache: " << (_use_disk_dds_cache?"true":"false") << endl);
415 _disk_meta_cache_path = get_beskeys("H5.DiskMetaDataCachePath");
416
417 key_value = obtain_beskeys_info("H5.EnableEOSGeoCacheFile",has_key);
418 if(has_key)
419 _use_latlon_disk_cache = key_value;
420 BESDEBUG(HDF5_NAME, prolog << "H5.EnableEOSGeoCacheFile: " << (_use_latlon_disk_cache?"true":"false") << endl);
421 _latlon_disk_cache_size = get_uint_key("H5.Cache.latlon.size",0);
422 _latlon_disk_cache_dir = get_beskeys("H5.Cache.latlon.path");
423 _latlon_disk_cachefile_prefix= get_beskeys("H5.Cache.latlon.prefix");
424
425 key_value = obtain_beskeys_info("H5.EscapeUTF8Attr", has_key);
426 if (has_key)
427 _escape_utf8_attr = key_value;
428
429#if 0
430 //_default_handle_dimension = check_and_set_beskeys("H5.DefaultHandleDimension");
431 key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
432 if(has_key)
433 _default_handle_dimension = key_value;
434 BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
435#endif
436
437 if(get_usecf()) {
438 if(get_lrdcache_entries()) {
439 lrdata_mem_cache = new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
440 bool has_LFMC_config = false;
441 key_value = obtain_beskeys_info("H5.LargeDataMemCacheConfig",has_key);
442 if(has_key)
443 has_LFMC_config = key_value;
444 BESDEBUG(HDF5_NAME, prolog << "H5.LargeDataMemCacheConfig: " << (has_LFMC_config?"true":"false") << endl);
445 if(true == has_LFMC_config) {
446 _common_cache_dirs =obtain_lrd_common_cache_dirs();
447#if 0
448 if(false == _common_cache_dirs)
449cerr<<"No specific cache info"<<endl;
450#endif
451 }
452 }
453 if(get_srdcache_entries()) {
454
455 BESDEBUG(HDF5_NAME, prolog << "Generate memory cache for smaller coordinate variables" << endl);
456 srdata_mem_cache = new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
457
458 }
459
460 if(_disk_cache_comp_data == true && _use_disk_cache == true) {
461 if(_disk_cache_comp_threshold < 1.0) {
462 ostringstream ss;
463 ss<< _disk_cache_comp_threshold;
464 string _comp_threshold_str(ss.str());
465 string invalid_comp_threshold ="The Compression Threshold is the total size of the variable array";
466 invalid_comp_threshold+=" divided by the storage size of compressed array. It should always be >1";
467 invalid_comp_threshold+=" The current threhold set at h5.conf is ";
468 invalid_comp_threshold+=_comp_threshold_str;
469 invalid_comp_threshold+=" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
470 throw BESInternalError(invalid_comp_threshold,__FILE__,__LINE__);
471 }
472 }
473 _stp_east_filename = get_beskeys("H5.STPEastFileName");
474 _stp_north_filename = get_beskeys("H5.STPNorthFileName");
475 }
476 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
477}
478
479
480// Build DAS
481bool HDF5RequestHandler::hdf5_build_das(BESDataHandlerInterface & dhi)
482{
483 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
484#if DYNAMIC_CONFIG_ENABLED
485 load_config();
486#endif
487
488 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
489 hid_t cf_fileid = -1;
490
491 // Obtain the HDF5 file name.
492 string filename = dhi.container->access();
493
494 // Obtain the BES object from the client
495 BESResponseObject *response = dhi.response_handler->get_response_object() ;
496
497 // Convert to the BES DAS response
498 BESDASResponse *bdas = dynamic_cast < BESDASResponse * >(response) ;
499 if( !bdas )
500 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
501
502 try {
504 DAS *das = bdas->get_das();
505
506 // Look inside the memory cache to see if it's initialized
507 DAS *cached_das_ptr = nullptr;
508 bool use_das_cache = false;
509 if (das_cache)
510 cached_das_ptr = static_cast<DAS*>(das_cache->get(filename));
511 if (cached_das_ptr)
512 use_das_cache = true;
513
514 if (true == use_das_cache) {
515
516 // copy the cached DAS into the BES response object
517 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
518 *das = *cached_das_ptr;
519 }
520 else {
521
522 bool das_from_dc = false;
523 string das_cache_fname;
524
525 // If the use_disk_meta_cache is set, check if the cache file exists and sets the flag.
526 if(_use_disk_meta_cache == true) {
527
528 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
529 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
530
531 if(access(das_cache_fname.c_str(),F_OK) !=-1)
532 das_from_dc = true;
533
534 }
535
536 // If reading DAS from the disk cache, call the corresponding function
537 if(true == das_from_dc) {
538 read_das_from_disk_cache(das_cache_fname,das);
539
540 // If the memory cache is set, adding the DAS copy to the memory cache
541 if (das_cache) {
542 // add a copy
543 BESDEBUG(HDF5_NAME, prolog << "HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
544 das_cache->add(new DAS(*das), filename);
545 }
546 }
547
548 else {// Need to build from the HDF5 file
549 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
550 if (true == _usecf) {//CF option
551
552 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
553 if (cf_fileid < 0){
554 string invalid_file_msg="Could not open this HDF5 file ";
555 invalid_file_msg +=filename;
556 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
557 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
558 invalid_file_msg +=" distributor.";
559 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
560 }
561 // Need to check if DAP4 DMR CF 64-bit integer mapping is on.
562 if(HDF5RequestHandler::get_dmr_64bit_int()!=nullptr)
563 HDF5RequestHandler::set_dmr_64bit_int(nullptr);
564 read_cfdas( *das,filename,cf_fileid);
565 H5Fclose(cf_fileid);
566 }
567 else {// Default option
568 hid_t fileid = get_fileid(filename.c_str());
569 if (fileid < 0) {
570 string invalid_file_msg="Could not open this HDF5 file ";
571 invalid_file_msg +=filename;
572 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
573 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
574 invalid_file_msg +=" distributor.";
575 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
576
577 }
578
579 find_gloattr(fileid, *das);
580 depth_first(fileid, "/", *das);
581 close_fileid(fileid);
582 }
583
584 Ancillary::read_ancillary_das( *das, filename ) ;
585
586#if 0
587// Dump all attribute contents
588AttrTable* top_table = das->get_top_level_attributes();
589get_attr_contents(top_table);
590
591// Dump all variable contents
592AttrTable::Attr_iter start_aiter=das->var_begin();
593AttrTable::Attr_iter it = start_aiter;
594AttrTable::Attr_iter end_aiter = das->var_end();
595while(it != end_aiter) {
596AttrTable* temp_table = das->get_table(it);
597if(temp_table!=0){
598cerr<<"var_begin"<<endl;
599temp_table->print(cerr);
600}
601++it;
602}
603#endif
604 // If the memory cache is turned on
605 if(das_cache) {
606 // add a copy
607 BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
608 das_cache->add(new DAS(*das), filename);
609 }
610
611 // DAS disk cache fname will be set only when the metadata disk cache is turned on
612 // So if it comes here, the das cache should be generated.
613 if(das_cache_fname!="") {
614 BESDEBUG(HDF5_NAME, prolog << "HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
615 write_das_to_disk_cache(das_cache_fname,das);
616 }
617 }
618 }
619
620 bdas->clear_container() ;
621 }
622 catch(BESError & e) {
623 if(cf_fileid !=-1)
624 H5Fclose(cf_fileid);
625 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
626 throw;
627 }
628 catch(InternalErr & e) {
629
630 if(cf_fileid !=-1)
631 H5Fclose(cf_fileid);
632 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
633 __FILE__, __LINE__);
634 }
635 catch(Error & e) {
636
637 if(cf_fileid !=-1)
638 H5Fclose(cf_fileid);
639 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
640 __FILE__, __LINE__);
641 }
642 catch(...) {
643
644 if(cf_fileid !=-1)
645 H5Fclose(cf_fileid);
646 string s = "unknown exception caught building HDF5 DAS";
647 throw BESInternalFatalError(s, __FILE__, __LINE__);
648 }
649
650 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
651 return true;
652}
653
654// Convenient function that helps build DDS and Data
655// Since this function will be used by both the DDS and Data services, we need to pass both BESDDSResponse and BESDataDDSResponse.
656// This two parameters are necessary for the future DDS disk cache feature.
657void HDF5RequestHandler::get_dds_with_attributes( BESDDSResponse*bdds,BESDataDDSResponse*data_bdds,const string &container_name, const string& filename,const string &dds_cache_fname, const string &das_cache_fname,bool dds_from_dc,bool das_from_dc, bool build_data)
658{
659 DDS *dds;
660 if(true == build_data)
661 dds = data_bdds->get_dds();
662 else dds = bdds->get_dds();
663
664 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
665 hid_t fileid = -1;
666 hid_t cf_fileid = -1;
667
668 try {
669
670 // Look in memory cache to see if it's initialized
671 DDS* cached_dds_ptr = nullptr;
672 bool use_dds_cache = false;
673 if (dds_cache)
674 cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename));
675 if (cached_dds_ptr)
676 use_dds_cache = true;
677 if (true == use_dds_cache) {
678 // copy the cached DDS into the BES response object. Assume that any cached DDS
679 // includes the DAS information.
680 BESDEBUG(HDF5_NAME, prolog << "DDS Metadata Cached hit for : " << filename << endl);
681 *dds = *cached_dds_ptr; // Copy the referenced object
682 }
683 else if (true ==dds_from_dc) {//Currently the dds_from_ds is always false by default.
684 read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
685 }
686 else {
687 BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
688 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
689 dds->filename(filename);
690
691 // For the time being, not mess up CF's fileID with Default's fileID
692 if(true == _usecf) {
693
694 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
695 if (cf_fileid < 0){
696 string invalid_file_msg="Could not open this HDF5 file ";
697 invalid_file_msg +=filename;
698 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
699 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
700 invalid_file_msg +=" distributor.";
701 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
702 }
703 // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
704 // to let the handler map the 64-bit integer.
705 if(HDF5RequestHandler::get_dmr_64bit_int() != nullptr)
706 HDF5RequestHandler::set_dmr_64bit_int(nullptr);
707 read_cfdds(*dds,filename,cf_fileid);
708 }
709 else {
710
711 fileid = get_fileid(filename.c_str());
712 if (fileid < 0) {
713 string invalid_file_msg="Could not open this HDF5 file ";
714 invalid_file_msg +=filename;
715 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
716 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
717 invalid_file_msg +=" distributor.";
718 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
719 }
720
721 depth_first(fileid, (char*)"/", *dds, filename.c_str());
722
723 }
724 // Check semantics
725 if (!dds->check_semantics()) { // DDS didn't comply with the semantics
726 dds->print(cerr);
727 throw InternalErr(__FILE__, __LINE__,
728 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
729 }
730
731 Ancillary::read_ancillary_dds( *dds, filename ) ;
732
733 // Generate the DDS cached file if needed,currently this is always false by default
734 if(dds_cache_fname!="" && dds_from_dc == false)
735 write_dds_to_disk_cache(dds_cache_fname,dds);
736
737 // Add attributes
738 {
739 hid_t h5_fd = -1;
740 if(_usecf == true)
741 h5_fd = cf_fileid;
742 else
743 h5_fd = fileid;
744 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
745 }
746
747 // Add memory cache if possible
748 if (dds_cache) {
749 // add a copy
750 BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
751 dds_cache->add(new DDS(*dds), filename);
752 }
753
754 if(cf_fileid != -1)
755 H5Fclose(cf_fileid);
756 if(fileid != -1)
757 H5Fclose(fileid);
758
759 }
760
761 }
762 catch(InternalErr & e) {
763
764 if(cf_fileid !=-1)
765 H5Fclose(cf_fileid);
766
767 if(fileid != -1)
768 H5Fclose(fileid);
769
770 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
771 __FILE__, __LINE__);
772 }
773 catch(Error & e) {
774
775 if(cf_fileid !=-1)
776 H5Fclose(cf_fileid);
777 if(fileid !=-1)
778 H5Fclose(fileid);
779
780 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
781 __FILE__, __LINE__);
782 }
783 catch(...) {
784
785 if(cf_fileid !=-1)
786 H5Fclose(cf_fileid);
787 if(fileid !=-1)
788 H5Fclose(fileid);
789
790 string s = "unknown exception caught building HDF5 DDS";
791 throw BESInternalFatalError(s, __FILE__, __LINE__);
792 }
793
794}
795
796void HDF5RequestHandler::get_dds_without_attributes_datadds(BESDataDDSResponse*data_bdds,const string &container_name, const string& filename)
797{
798 DDS *dds = data_bdds->get_dds();
799
800 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
801 hid_t fileid = -1;
802 hid_t cf_fileid = -1;
803
804 try {
805
806 // Look in memory cache to see if it's initialized
807 DDS* cached_dds_ptr = nullptr;
808 bool use_datadds_cache = false;
809 if (datadds_cache)
810 cached_dds_ptr = static_cast<DDS*>(datadds_cache->get(filename));
811 if (cached_dds_ptr)
812 use_datadds_cache = true;
813 if (true == use_datadds_cache) {
814 // copy the cached DDS into the BES response object.
815 // The DAS information is not included.
816 BESDEBUG(HDF5_NAME, prolog << "DataDDS Metadata Cached hit for : " << filename << endl);
817 *dds = *cached_dds_ptr; // Copy the referenced object
818 }
819 else {
820 BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
821 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
822 dds->filename(filename);
823
824 // For the time being, not mess up CF's fileID with Default's fileID
825 if(true == _usecf) {
826
827 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
828 if (cf_fileid < 0){
829 string invalid_file_msg="Could not open this HDF5 file ";
830 invalid_file_msg +=filename;
831 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
832 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
833 invalid_file_msg +=" distributor.";
834 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
835 }
836 // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
837 // to let the handler map the 64-bit integer.
838 if(HDF5RequestHandler::get_dmr_64bit_int() != nullptr)
839 HDF5RequestHandler::set_dmr_64bit_int(nullptr);
840 read_cfdds(*dds,filename,cf_fileid);
841 }
842 else {
843
844 fileid = get_fileid(filename.c_str());
845 if (fileid < 0) {
846 string invalid_file_msg="Could not open this HDF5 file ";
847 invalid_file_msg +=filename;
848 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
849 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
850 invalid_file_msg +=" distributor.";
851 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
852 }
853
854 depth_first(fileid, (char*)"/", *dds, filename.c_str());
855
856 }
857 // Check semantics
858 if (!dds->check_semantics()) { // DDS didn't comply with the semantics
859 dds->print(cerr);
860 throw InternalErr(__FILE__, __LINE__,
861 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
862 }
863
864 Ancillary::read_ancillary_dds( *dds, filename ) ;
865
866#if 0
867 // Add attributes
868 {
869 hid_t h5_fd = -1;
870 if(_usecf == true)
871 h5_fd = cf_fileid;
872 else
873 h5_fd = fileid;
874 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
875 }
876#endif
877
878 // Add memory cache if possible
879 if (datadds_cache) {
880 // add a copy
881 BESDEBUG(HDF5_NAME, prolog << "DataDDS added to the cache for : " << filename << endl);
882 datadds_cache->add(new DDS(*dds), filename);
883 }
884
885 if(cf_fileid != -1)
886 H5Fclose(cf_fileid);
887 if(fileid != -1)
888 H5Fclose(fileid);
889
890 }
891 BESDEBUG(HDF5_NAME, prolog << "Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
892 data_bdds->set_ia_flag(false);
893
894 }
895 catch(InternalErr & e) {
896
897 if(cf_fileid !=-1)
898 H5Fclose(cf_fileid);
899
900 if(fileid != -1)
901 H5Fclose(fileid);
902
903 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
904 __FILE__, __LINE__);
905 }
906 catch(Error & e) {
907
908 if(cf_fileid !=-1)
909 H5Fclose(cf_fileid);
910 if(fileid !=-1)
911 H5Fclose(fileid);
912
913 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
914 __FILE__, __LINE__);
915 }
916 catch(...) {
917
918 if(cf_fileid !=-1)
919 H5Fclose(cf_fileid);
920 if(fileid !=-1)
921 H5Fclose(fileid);
922
923 string s = "unknown exception caught building HDF5 DDS";
924 throw BESInternalFatalError(s, __FILE__, __LINE__);
925 }
926
927}
928
929
930#if 0
931// OLD function: Keep it for a while
932// Convenient function that helps build DDS and Data
933void HDF5RequestHandler::get_dds_with_attributes(const string &filename, const string &container_name, DDS*dds) {
934
935 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
936 hid_t fileid = -1;
937 hid_t cf_fileid = -1;
938
939 try {
940
941 // Look in memory cache to see if it's initialized
942 DDS* cached_dds_ptr = 0;
943 if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename)))) {
944 // copy the cached DDS into the BES response object. Assume that any cached DDS
945 // includes the DAS information.
946 BESDEBUG(HDF5_NAME, prolog << "DDS Cached hit for : " << filename << endl);
947 *dds = *cached_dds_ptr; // Copy the referenced object
948 }
949
950 else {
951
952 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
953 if (!container_name.empty())
954 dds->container_name(container_name);
955 dds->filename(filename);
956
957 // For the time being, not mess up CF's fileID with Default's fileID
958 if(true == _usecf) {
959// This block cannot be used to cache the data
960#if 0
961 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
962 string dds_filename = "/tmp/"+base_filename+"_dds";
963 FILE *dds_file = fopen(dds_filename.c_str(),"r");
964cerr<<"before parsing "<<endl;
966DDS tdds(&tf,name_path(filename),"3.2");
967tdds.filename(filename);
968 //dds->parse(dds_file);
969 tdds.parse(dds_file);
970 //DDS *cache_dds = new DDS(tdds);
971 cache_dds = new DDS(tdds);
972if(dds!=nullptr)
973 delete dds;
974dds = cache_dds;
975tdds.print(cout);
976dds->print(cout);
977cerr<<"after parsing "<<endl;
978//dds->print(cout);
979 // fclose(dds_file);
980//#endif
981
982#endif
983// end of this block
984
985
986 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
987 if (cf_fileid < 0){
988 string invalid_file_msg="Could not open this HDF5 file ";
989 invalid_file_msg +=filename;
990 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
991 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
992 invalid_file_msg +=" distributor.";
993 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
994 }
995//#if 0
996 read_cfdds(*dds,filename,cf_fileid);
997//#endif
998 // Generate the DDS cached file
999 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1000 string dds_filename = "/tmp/"+base_filename+"_dds";
1001 FILE *dds_file = fopen(dds_filename.c_str(),"w");
1002 dds->print(dds_file);
1003 fclose(dds_file);
1004
1005 }
1006 else {
1007
1008 fileid = get_fileid(filename.c_str());
1009 if (fileid < 0) {
1010 string invalid_file_msg="Could not open this HDF5 file ";
1011 invalid_file_msg +=filename;
1012 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1013 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1014 invalid_file_msg +=" distributor.";
1015 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1016 }
1017
1018 depth_first(fileid, (char*)"/", *dds, filename.c_str());
1019
1020 }
1021
1022
1023 // Check semantics
1024 if (!dds->check_semantics()) { // DDS didn't comply with the semantics
1025 dds->print(cerr);
1026 throw InternalErr(__FILE__, __LINE__,
1027 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
1028 }
1029
1030 Ancillary::read_ancillary_dds( *dds, filename ) ;
1031
1032
1033 // Check DAS cache
1034 DAS *das = 0 ;
1035
1036 if (das_cache && (das = static_cast<DAS*>(das_cache->get(filename)))) {
1037 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
1038 dds->transfer_attributes(das); // no need to copy the cached DAS
1039 }
1040
1041 else {
1042
1043 das = new DAS ;
1044
1045 if (!container_name.empty())
1046 das->container_name(container_name);
1047
1048 if (true == _usecf) {
1049
1050 // go to the CF option
1051 read_cfdas( *das,filename,cf_fileid);
1052
1053 }
1054 else {
1055
1056 find_gloattr(fileid, *das);
1057 depth_first(fileid, "/", *das);
1058 close_fileid(fileid);
1059 }
1060
1061 if(cf_fileid != -1)
1062 H5Fclose(cf_fileid);
1063
1064 Ancillary::read_ancillary_das( *das, filename ) ;
1065
1066 dds->transfer_attributes(das);
1067
1068
1069 // Only free the DAS if it's not added to the cache
1070 if (das_cache) {
1071 // add a copy
1072 BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
1073 //das_cache->add(new DAS(*das), filename);
1074 das_cache->add(das, filename);
1075 }
1076 else {
1077 delete das;
1078 }
1079 }
1080
1081 if (dds_cache) {
1082 // add a copy
1083 BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
1084 dds_cache->add(new DDS(*dds), filename);
1085 }
1086
1087 }
1088
1089//dds->print(cout);
1090
1091 }
1092 catch(InternalErr & e) {
1093
1094 if(cf_fileid !=-1)
1095 H5Fclose(cf_fileid);
1096
1097 if(fileid != -1)
1098 H5Fclose(fileid);
1099
1100 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1101 __FILE__, __LINE__);
1102 }
1103 catch(Error & e) {
1104
1105 if(cf_fileid !=-1)
1106 H5Fclose(cf_fileid);
1107 if(fileid !=-1)
1108 H5Fclose(fileid);
1109
1110 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1111 __FILE__, __LINE__);
1112 }
1113 catch(...) {
1114
1115 if(cf_fileid !=-1)
1116 H5Fclose(cf_fileid);
1117 if(fileid !=-1)
1118 H5Fclose(fileid);
1119
1120 string s = "unknown exception caught building HDF5 DDS";
1121 throw BESInternalFatalError(s, __FILE__, __LINE__);
1122 }
1123
1124}
1125#endif
1126
1127// Build DDS
1128bool HDF5RequestHandler::hdf5_build_dds(BESDataHandlerInterface & dhi)
1129{
1130 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1131#if DYNAMIC_CONFIG_ENABLED
1132 load_config();
1133#endif
1134
1135 // Obtain the HDF5 file name.
1136 string filename = dhi.container->access();
1137
1138 string container_name = dhi.container->get_symbolic_name();
1139 BESResponseObject *response = dhi.response_handler->get_response_object();
1140 BESDDSResponse *bdds = dynamic_cast < BESDDSResponse * >(response);
1141 if( !bdds )
1142 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1143 bdds->set_container(container_name);
1144
1145 try {
1146
1147 bool dds_from_dc = false;
1148 bool das_from_dc = false;
1149 bool build_data = false;
1150 string dds_cache_fname;
1151 string das_cache_fname;
1152
1153 if(_use_disk_meta_cache) {
1154
1155 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1156
1157 // The _use_disk_dds_cache is always set to false by default
1158 if(_use_disk_dds_cache) {
1159 dds_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_dds";
1160 if(access(dds_cache_fname.c_str(),F_OK) !=-1)
1161 dds_from_dc = true;
1162 }
1163
1164 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1165 // Check if das files exist
1166 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1167 das_from_dc = true;
1168
1169 }
1170
1171 get_dds_with_attributes(bdds, nullptr,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1172
1173 // The following block reads dds from a dds cache file.
1174#if 0
1175 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1176 string dds_filename = "/tmp/"+base_filename+"_dds";
1177
1178 BaseTypeFactory tf;
1179 DDS tdds(&tf,name_path(filename),"3.2");
1180 tdds.filename(filename);
1181
1182
1183 FILE *dds_file = fopen(dds_filename.c_str(),"r");
1184 tdds.parse(dds_file);
1185//cerr<<"before parsing "<<endl;
1186 DDS* cache_dds = new DDS(tdds);
1187 if(dds != nullptr)
1188 delete dds;
1189 bdds->set_dds(cache_dds);
1190 fclose(dds_file);
1191#endif
1192
1193 bdds->set_constraint( dhi ) ;
1194 bdds->clear_container() ;
1195
1196 }
1197 catch(BESError & e) {
1198 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1199 throw;
1200 }
1201 catch(InternalErr & e) {
1202
1203 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1204 __FILE__, __LINE__);
1205 }
1206 catch(Error & e) {
1207
1208 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1209 __FILE__, __LINE__);
1210 }
1211 catch(...) {
1212
1213 string s = "unknown exception caught building HDF5 DDS";
1214 throw BESInternalFatalError(s, __FILE__, __LINE__);
1215 }
1216
1217 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1218 return true;
1219}
1220
1221bool HDF5RequestHandler::hdf5_build_data(BESDataHandlerInterface & dhi)
1222{
1223 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1224#if DYNAMIC_CONFIG_ENABLED
1225 load_config();
1226#endif
1227
1228 if(true ==_usecf) {
1229
1230 if(true == _pass_fileid)
1231 return hdf5_build_data_with_IDs(dhi);
1232 }
1233
1234 string filename = dhi.container->access();
1235
1236 string container_name = dhi.container->get_symbolic_name();
1237 BESResponseObject *response = dhi.response_handler->get_response_object();
1238 BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1239 if( !bdds )
1240 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1241 bdds->set_container(container_name);
1242
1243 try {
1244
1245 // DDS from disk cache is not currently supported. It may be supported in the future.
1246 // We will leave the code as commented.
1247#if 0
1248 bool dds_from_dc = false;
1249 bool build_data = true;
1250#endif
1251
1252 bool das_from_dc = false;
1253 string dds_cache_fname;
1254 string das_cache_fname;
1255
1256
1257 // Only DAS is read from the cache. dds_from_dc is always false.
1258 if(_use_disk_meta_cache == true) {
1259
1260 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1261 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1262
1263 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1264 das_from_dc = true;
1265
1266 }
1267
1268#if 0
1269 get_dds_with_attributes(nullptr,bdds, container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1270#endif
1271 get_dds_without_attributes_datadds(bdds,container_name,filename);
1272
1273 bdds->set_constraint( dhi ) ;
1274 bdds->clear_container() ;
1275
1276 }
1277 catch(BESError & e) {
1278 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1279 throw;
1280 }
1281 catch(InternalErr & e) {
1282
1283 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1284 __FILE__, __LINE__);
1285 }
1286 catch(Error & e) {
1287
1288 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1289 __FILE__, __LINE__);
1290 }
1291 catch(...) {
1292
1293 string s = "unknown exception caught building HDF5 DDS";
1294 throw BESInternalFatalError(s, __FILE__, __LINE__);
1295 }
1296
1297 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1298 return true;
1299}
1300
1301// Obtain data when turning on the pass fileID key.The memory cache is not used.
1302bool HDF5RequestHandler::hdf5_build_data_with_IDs(BESDataHandlerInterface & dhi)
1303{
1304 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1305#if DYNAMIC_CONFIG_ENABLED
1306 load_config();
1307#endif
1308
1309 BESDEBUG(HDF5_NAME,prolog << "Building DataDDS by passing file IDs. "<<endl);
1310 hid_t cf_fileid = -1;
1311
1312 string filename = dhi.container->access();
1313
1314 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1315 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1316 if (cf_fileid < 0){
1317 string invalid_file_msg="Could not open this HDF5 file ";
1318 invalid_file_msg +=filename;
1319 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1320 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1321 invalid_file_msg +=" distributor.";
1322 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1323 }
1324
1325 BESResponseObject *response = dhi.response_handler->get_response_object();
1326 BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1327 if( !bdds )
1328 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1329
1330 try {
1331
1332 bdds->set_container( dhi.container->get_symbolic_name() ) ;
1333
1334 auto hdds = new HDF5DDS(bdds->get_dds());
1335 delete bdds->get_dds();
1336
1337 bdds->set_dds(hdds);
1338
1339 hdds->setHDF5Dataset(cf_fileid);
1340
1341 read_cfdds( *hdds,filename,cf_fileid);
1342
1343 if (!hdds->check_semantics()) { // DDS didn't comply with the DAP semantics
1344 hdds->print(cerr);
1345 throw InternalErr(__FILE__, __LINE__,
1346 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1347 }
1348
1349 Ancillary::read_ancillary_dds( *hdds, filename ) ;
1350
1351 DAS *das = new DAS ;
1352 BESDASResponse bdas( das ) ;
1354 read_cfdas( *das,filename,cf_fileid);
1355 Ancillary::read_ancillary_das( *das, filename ) ;
1356
1357 hdds->transfer_attributes(das);
1358 bdds->set_constraint( dhi ) ;
1359 bdds->clear_container() ;
1360
1361 }
1362
1363 catch(BESError & e) {
1364 if(cf_fileid !=-1)
1365 H5Fclose(cf_fileid);
1366 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1367 throw;
1368 }
1369 catch(InternalErr & e) {
1370 if(cf_fileid !=-1)
1371 H5Fclose(cf_fileid);
1372 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1373 __FILE__, __LINE__);
1374 }
1375 catch(Error & e) {
1376 if(cf_fileid !=-1)
1377 H5Fclose(cf_fileid);
1378 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1379 __FILE__, __LINE__);
1380 }
1381 catch(...) {
1382 if(cf_fileid !=-1)
1383 H5Fclose(cf_fileid);
1384 string s = "unknown exception caught building HDF5 DataDDS";
1385 throw BESInternalFatalError(s, __FILE__, __LINE__);
1386 }
1387
1388 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1389 return true;
1390}
1391
1392bool HDF5RequestHandler::hdf5_build_dmr(BESDataHandlerInterface & dhi)
1393{
1394 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1395#if DYNAMIC_CONFIG_ENABLED
1396 load_config();
1397#endif
1398
1399 // Extract the DMR Response object - this holds the DMR used by the
1400 // other parts of the framework.
1401 BESResponseObject *response = dhi.response_handler->get_response_object();
1402 BESDMRResponse &bes_dmr_response = dynamic_cast<BESDMRResponse &>(*response);
1403
1404 string filename = dhi.container->access();
1405
1406 DMR *dmr = bes_dmr_response.get_dmr();
1407
1408 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
1409 hid_t fileid = -1;
1410 hid_t cf_fileid = -1;
1411
1412 try {
1413
1414 DMR* cached_dmr_ptr = nullptr;
1415 if (dmr_cache){
1416 BESDEBUG(HDF5_NAME, prolog << "Checking DMR cache for : " << filename << endl);
1417 cached_dmr_ptr = static_cast<DMR*>(dmr_cache->get(filename));
1418 }
1419
1420 if (cached_dmr_ptr) {
1421 // copy the cached DMR into the BES response object
1422 BESDEBUG(HDF5_NAME, prolog << "DMR cache hit for : " << filename << endl);
1423 *dmr = *cached_dmr_ptr; // Copy the referenced object
1424 dmr->set_request_xml_base(bes_dmr_response.get_request_xml_base());
1425 }
1426 else {// No cache
1427
1428 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1429 D4BaseTypeFactory MyD4TypeFactory;
1430 dmr->set_factory(&MyD4TypeFactory);
1431
1432 if(true ==_usecf) {// CF option
1433
1434 if(true == _usecfdmr) {
1435 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1436 if (cf_fileid < 0){
1437 string invalid_file_msg="Could not open this HDF5 file ";
1438 invalid_file_msg +=filename;
1439 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1440 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1441 invalid_file_msg +=" distributor.";
1442 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1443 }
1444 read_cfdmr(dmr,filename,cf_fileid);
1445 H5Fclose(cf_fileid);
1446 bes_dmr_response.set_dap4_constraint(dhi);
1447 bes_dmr_response.set_dap4_function(dhi);
1448 dmr->set_factory(nullptr);
1449
1450 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1451
1452 return true;
1453 }
1454
1455 if(true == _pass_fileid)
1456 return hdf5_build_dmr_with_IDs(dhi);
1457
1458 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1459 if (cf_fileid < 0){
1460 string invalid_file_msg="Could not open this HDF5 file ";
1461 invalid_file_msg +=filename;
1462 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1463 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1464 invalid_file_msg +=" distributor.";
1465 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1466 }
1467
1468
1469 BaseTypeFactory factory;
1470 DDS dds(&factory, name_path(filename), "3.2");
1471 dds.filename(filename);
1472
1473 DAS das;
1474
1475 // For the CF option dmr response, we need to map 64-bit integer separately
1476 // So set the flag to map 64-bit integer.
1477 HDF5RequestHandler::set_dmr_64bit_int(dmr);
1478 read_cfdds( dds,filename,cf_fileid);
1479 if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1480 dds.print(cerr);
1481 throw InternalErr(__FILE__, __LINE__,
1482 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1483 }
1484
1485 read_cfdas(das,filename,cf_fileid);
1486 Ancillary::read_ancillary_das( das, filename ) ;
1487
1488 dds.transfer_attributes(&das);
1489
1491 if(cf_fileid !=-1)
1492 H5Fclose(cf_fileid);
1493
1494 dmr->build_using_dds(dds);
1495
1496 }// "if(true == _usecf)"
1497 else {// default option
1498
1499 // Obtain the HDF5 file ID.
1500 fileid = get_fileid(filename.c_str());
1501 if (fileid < 0) {
1502 string invalid_file_msg="Could not open this HDF5 file ";
1503 invalid_file_msg +=filename;
1504 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1505 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1506 invalid_file_msg +=" distributor.";
1507 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1508 }
1509
1510 bool use_dimscale = false;
1511 if(true == _default_handle_dimension)
1512 use_dimscale = check_dimscale(fileid);
1513 dmr->set_name(name_path(filename));
1514 dmr->set_filename(name_path(filename));
1515
1516 // The breadth_first() function builds the variables and attributes and
1517 // loads them into the root group (building child groups as needed).
1518 // jhrg 4/30/20
1519 D4Group* root_grp = dmr->root();
1520 BESDEBUG("h5", "use_dimscale is "<< use_dimscale <<endl);
1521
1522 // It is possible that a dimension variable has hardlinks. To make it
1523 // right for the netCDF-4 data model and the current DAP4 implementation,
1524 // we need to choose the shortest path of all hardlinks as the dimension path.
1525 // So to avoid iterate all HDF5 objects multiple times, save the found
1526 // hardlinks and search them when necessary. Note we have to search hardlinks from the root.
1527 // KY 2021-11-15
1528 vector<link_info_t> hdf5_hls;
1529 breadth_first(fileid, fileid,(char*)"/",root_grp,filename.c_str(),use_dimscale,hdf5_hls);
1530#if 0
1531 BESDEBUG("h5", "build_dmr - before obtain dimensions"<< endl);
1532 D4Dimensions *root_dims = root_grp->dims();
1533 for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1534 BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
1535 BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1536 BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1537 BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1538 //cout <<"dim size is: "<<(*di)->size()<<endl;
1539 //cout <<"dim fully_qualified_name is: "<<(*di)->fully_qualified_name()<<endl;
1540 }
1541 BESDEBUG("h5", "build_dmr - after obtain dimensions"<< endl);
1542#endif
1543
1544#if 0
1545 if(true == use_dimscale)
1546 //breadth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str(),true);
1547 breadth_first(fileid,(char*)"/",root_grp,filename.c_str(),true);
1548 else
1549 depth_first(fileid,(char*)"/",root_grp,filename.c_str());
1550 //depth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str());
1551#endif
1552
1553 close_fileid(fileid);
1554
1555 }// else (default option)
1556
1557 // If the cache is turned on, add the memory cache.
1558 if (dmr_cache) {
1559 // add a copy
1560 BESDEBUG(HDF5_NAME, prolog << "DMR added to the cache for : " << filename << endl);
1561 dmr_cache->add(new DMR(*dmr), filename);
1562 }
1563 }// else no cache
1564 }// try
1565 catch(BESError & e) {
1566 if(cf_fileid !=-1)
1567 H5Fclose(cf_fileid);
1568 if(fileid !=-1)
1569 H5Fclose(fileid);
1570 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1571 throw;
1572 }
1573 catch(InternalErr & e) {
1574
1575 if(cf_fileid !=-1)
1576 H5Fclose(cf_fileid);
1577 if(fileid !=-1)
1578 H5Fclose(fileid);
1579
1580 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1581 __FILE__, __LINE__);
1582 }
1583 catch(Error & e) {
1584
1585 if(cf_fileid !=-1)
1586 H5Fclose(cf_fileid);
1587 if(fileid !=-1)
1588 H5Fclose(fileid);
1589 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1590 __FILE__, __LINE__);
1591 }
1592 catch(...) {
1593
1594 if(cf_fileid !=-1)
1595 H5Fclose(cf_fileid);
1596 if(fileid !=-1)
1597 H5Fclose(fileid);
1598 string s = "unknown exception caught building HDF5 DMR";
1599 throw BESInternalFatalError(s, __FILE__, __LINE__);
1600 }
1601
1602#if 0
1603 //dmr->print(cout);
1604#endif
1605
1606 // Instead of fiddling with the internal storage of the DHI object,
1607 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1608 // methods to set the constraints. But, why? Ans: from Patrick is that
1609 // in the 'container' mode of BES each container can have a different
1610 // CE.
1611 bes_dmr_response.set_dap4_constraint(dhi);
1612 bes_dmr_response.set_dap4_function(dhi);
1613 dmr->set_factory(nullptr);
1614
1615 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1616 return true;
1617}
1618
1619// This function is only used when EnableCF is true.
1620bool HDF5RequestHandler::hdf5_build_dmr_with_IDs(BESDataHandlerInterface & dhi)
1621{
1622 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1623#if DYNAMIC_CONFIG_ENABLED
1624 load_config();
1625#endif
1626
1627 BESDEBUG("h5","Building DMR with passing file IDs. "<<endl);
1628 string filename = dhi.container->access();
1629 hid_t cf_fileid = -1;
1630
1631 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1632 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1633 if (cf_fileid < 0){
1634 string invalid_file_msg="Could not open this HDF5 file ";
1635 invalid_file_msg +=filename;
1636 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1637 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1638 invalid_file_msg +=" distributor.";
1639 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1640 }
1641
1642 BaseTypeFactory factory;
1643 DDS dds(&factory, name_path(filename), "3.2");
1644 dds.filename(filename);
1645
1646 DAS das;
1647
1648 try {
1649
1650
1651 // This is the CF option
1652 read_cfdds( dds,filename,cf_fileid);
1653
1654 if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1655 dds.print(cerr);
1656 throw InternalErr(__FILE__, __LINE__,
1657 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1658 }
1659
1660 Ancillary::read_ancillary_dds( dds, filename ) ;
1661
1662
1663 read_cfdas(das,filename,cf_fileid);
1664
1665 Ancillary::read_ancillary_das( das, filename ) ;
1666
1667 dds.transfer_attributes(&das);
1668
1670#if 0
1671 //if(cf_fileid !=-1)
1672 // H5Fclose(cf_fileid);
1673#endif
1674
1675 }
1676 catch(BESError & e) {
1677 if(cf_fileid !=-1)
1678 H5Fclose(cf_fileid);
1679 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1680 throw;
1681 }
1682 catch(InternalErr & e) {
1683
1684 if(cf_fileid !=-1)
1685 H5Fclose(cf_fileid);
1686
1687 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1688 __FILE__, __LINE__);
1689 }
1690 catch(Error & e) {
1691
1692 if(cf_fileid !=-1)
1693 H5Fclose(cf_fileid);
1694
1695 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1696 __FILE__, __LINE__);
1697 }
1698 catch(...) {
1699
1700 if(cf_fileid !=-1)
1701 H5Fclose(cf_fileid);
1702
1703 string s = "unknown exception caught building HDF5 DataDDS";
1704 throw BESInternalFatalError(s, __FILE__, __LINE__);
1705 }
1706
1707 // Extract the DMR Response object - this holds the DMR used by the
1708 // other parts of the framework.
1709 BESResponseObject *response = dhi.response_handler->get_response_object();
1710 BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1711
1712 // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1713 // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1714 // the BES built and add this one. The HDF4DMR object will close the open dataset when
1715 // the BES runs the DMR's destructor.
1716
1717 DMR *dmr = bes_dmr.get_dmr();
1718 D4BaseTypeFactory MyD4TypeFactory;
1719 dmr->set_factory(&MyD4TypeFactory);
1720 dmr->build_using_dds(dds);
1721
1722 auto hdf5_dmr = new HDF5DMR(dmr);
1723 hdf5_dmr->setHDF5Dataset(cf_fileid);
1724 delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1725 bes_dmr.set_dmr(hdf5_dmr); // BESDMRResponse will delete hdf5_dmr
1726
1727 // Instead of fiddling with the internal storage of the DHI object,
1728 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1729 // methods to set the constraints. But, why? Ans: from Patrick is that
1730 // in the 'container' mode of BES each container can have a different
1731 // CE.
1732 bes_dmr.set_dap4_constraint(dhi);
1733 bes_dmr.set_dap4_function(dhi);
1734 hdf5_dmr->set_factory(nullptr);
1735
1736 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1737 return true;
1738}
1739
1740bool HDF5RequestHandler::hdf5_build_help(BESDataHandlerInterface & dhi)
1741{
1742 BESResponseObject *response = dhi.response_handler->get_response_object();
1743 BESInfo *info = dynamic_cast<BESInfo *>(response);
1744 if( !info )
1745 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1746
1747 string add_info="Just for Test";
1748
1749 map<string,string> attrs ;
1750 attrs["name"] = MODULE_NAME ;
1751 attrs["version"] = MODULE_VERSION ;
1752 list<string> services ;
1753 BESServiceRegistry::TheRegistry()->services_handled( HDF5_NAME, services );
1754 if( services.empty()==false )
1755 {
1756 string handles = BESUtil::implode( services, ',' ) ;
1757 attrs["handles"] = handles ;
1758 }
1759 info->begin_tag( "module", &attrs ) ;
1760 info->end_tag( "module" ) ;
1761 info->add_data(add_info);
1762
1763 return true;
1764}
1765
1766bool HDF5RequestHandler::hdf5_build_version(BESDataHandlerInterface & dhi)
1767{
1768 BESResponseObject *response = dhi.response_handler->get_response_object();
1769 BESVersionInfo *info = dynamic_cast < BESVersionInfo * >(response);
1770 if( !info )
1771 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1772
1773 info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1774
1775 return true;
1776}
1777
1778
1779bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1780{
1781 string lrd_config_fpath;
1782 string lrd_config_fname;
1783
1784 // Obtain DataCache path
1785 lrd_config_fpath = get_beskeys("H5.DataCachePath");
1786
1787 // Obtain the configure file name that specifics the large file configuration
1788 lrd_config_fname = get_beskeys("H5.LargeDataMemCacheFileName");
1789
1790 // If either the configure file path or fname is missing, won't add specific mem. cache dirs.
1791 if(lrd_config_fpath=="" || lrd_config_fname=="")
1792 return false;
1793
1794 // temp_line for storing info of one line in the config. file
1795 string temp_line;
1796
1797 // The full path of the configure file
1798 string mcache_config_fname = lrd_config_fpath+"/"+lrd_config_fname;
1799
1800#if 0
1801 //ifstream mcache_config_file("example.txt");
1802#endif
1803 // Open the configure file
1804 ifstream mcache_config_file(mcache_config_fname.c_str());
1805
1806 // If the configuration file is not open, return false.
1807 if(mcache_config_file.is_open()==false){
1808 BESDEBUG(HDF5_NAME, prolog << "The large data memory cache configure file "<<mcache_config_fname );
1809 BESDEBUG(HDF5_NAME, prolog << " cannot be opened."<<endl);
1810 return false;
1811 }
1812
1813 // Read the configuration file line by line
1814 while(getline(mcache_config_file,temp_line)) {
1815
1816 // Only consider lines that is no less than 2 characters and the 2nd character is space.
1817 if(temp_line.size()>1 && temp_line.at(1)==' ') {
1818 char sep=' ';
1819 string subline = temp_line.substr(2);
1820 vector<string> temp_name_list;
1821
1822 // Include directories to store common latitude and longitude values
1823 if(temp_line.at(0)=='1') {
1824 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1825#if 0
1826 //lrd_cache_dir_list +=temp_name_list;
1827#endif
1828 lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1829 }
1830 // Include directories not to store common latitude and longitude values
1831 else if(temp_line.at(0)=='0'){
1832 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1833#if 0
1834 //lrd_non_cache_dir_list +=temp_name_list;
1835#endif
1836 lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1837 }
1838 // Include variable names that the server would like to store in the memory cache
1839 else if(temp_line.at(0)=='2') {
1840
1841 // We need to handle the space case inside a variable path
1842 // either "" or '' needs to be used to identify a var path
1843 vector<int>dq_pos;
1844 vector<int>sq_pos;
1845 for(unsigned int i = 0; i<subline.size();i++){
1846 if(subline[i]=='"') {
1847 dq_pos.push_back(i);
1848 }
1849 else if(subline[i]=='\'')
1850 sq_pos.push_back(i);
1851 }
1852 if(dq_pos.empty() && sq_pos.empty())
1853 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1854 else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1855 unsigned int dq_index= 0;
1856 while(dq_index < dq_pos.size()){
1857 if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1858 temp_name_list.push_back
1859 (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1860 }
1861 dq_index = dq_index + 2;
1862 }
1863 }
1864 else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1865 unsigned int sq_index= 0;
1866 while(sq_index < sq_pos.size()){
1867 if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1868 temp_name_list.push_back
1869 (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1870 }
1871 sq_index = sq_index+2;
1872 }
1873 }
1874
1875 lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1876 }
1877 }
1878 }
1879
1880
1881#if 0
1882
1883for(int i =0; i<lrd_cache_dir_list.size();i++)
1884cerr<<"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1885for(int i =0; i<lrd_non_cache_dir_list.size();i++)
1886cerr<<"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1887for(int i =0; i<lrd_var_cache_file_list.size();i++)
1888cerr<<"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1889#endif
1890
1891
1892 mcache_config_file.close();
1893 if(lrd_cache_dir_list.empty() && lrd_non_cache_dir_list.empty() && lrd_var_cache_file_list.empty())
1894 return false;
1895 else
1896 return true;
1897}
1898
1899
1900bool HDF5RequestHandler::read_das_from_disk_cache(const string & cache_filename,DAS *das_ptr) {
1901
1902 BESDEBUG(HDF5_NAME, prolog << "Coming to read_das_from_disk_cache() " << cache_filename << endl);
1903 bool ret_value = true;
1904 FILE *md_file = nullptr;
1905 md_file = fopen(cache_filename.c_str(),"rb");
1906
1907 if(nullptr == md_file) {
1908 string bes_error = "An error occurred trying to open a metadata cache file " + cache_filename;
1909 throw BESInternalError( bes_error, __FILE__, __LINE__);
1910 }
1911 else {
1912
1913 int fd_md = fileno(md_file);
1914 struct flock *l_md;
1915 l_md = lock(F_RDLCK);
1916
1917 // hold a read(shared) lock to read metadata from a file.
1918 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1919 fclose(md_file);
1920 ostringstream oss;
1921 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1922 throw BESInternalError( oss.str(), __FILE__, __LINE__);
1923 }
1924
1925 try {
1926
1927 struct stat sb;
1928 if(stat(cache_filename.c_str(),&sb) != 0) {
1929 string bes_error = "An error occurred trying to stat a metadata cache file size " + cache_filename;
1930 throw BESInternalError( bes_error, __FILE__, __LINE__);
1931
1932 }
1933
1934
1935 auto bytes_expected_read=(size_t)sb.st_size;
1936 BESDEBUG(HDF5_NAME, prolog << "DAS Disk cache file size is " << bytes_expected_read << endl);
1937
1938 vector<char> buf;
1939 buf.resize(bytes_expected_read);
1940 size_t bytes_to_read =fread((void*)buf.data(),1,bytes_expected_read,md_file);
1941 if(bytes_to_read != bytes_expected_read)
1942 throw InternalErr(__FILE__,__LINE__,"Fail to read the data from the das cache file.");
1943
1944 char* temp_pointer =buf.data();
1945
1946 AttrTable*at = nullptr;
1947 // recursively build DAS
1948//#if 0
1949 temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1950//#endif
1951
1952
1953 }
1954 catch(...) {
1955 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1956 fclose(md_file);
1957 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1958 }
1959
1960 fclose(md_file);
1961 throw InternalErr(__FILE__,__LINE__,"Fail to parse a das cache file.");
1962 }
1963
1964 // Unlock the cache file
1965 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1966 fclose(md_file);
1967 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1968 }
1969 fclose(md_file);
1970 }
1971 return ret_value;
1972
1973}
1974
1975// This fucntion will NOT be used by default. Leave here for future improvement.
1976bool HDF5RequestHandler::write_dds_to_disk_cache(const string& dds_cache_fname,DDS *dds_ptr) {
1977
1978 BESDEBUG(HDF5_NAME, prolog << "Write DDS to disk cache " << dds_cache_fname << endl);
1979 FILE *dds_file = fopen(dds_cache_fname.c_str(),"w");
1980
1981 if(nullptr == dds_file) {
1982 string bes_error = "An error occurred trying to open a metadata cache file " + dds_cache_fname;
1983 throw BESInternalError( bes_error, __FILE__, __LINE__);
1984 }
1985 else {
1986
1987 int fd_md = fileno(dds_file);
1988 struct flock *l_md;
1989 l_md = lock(F_WRLCK);
1990
1991 // hold a read(shared) lock to read metadata from a file.
1992 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1993 fclose(dds_file);
1994 ostringstream oss;
1995 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1996 throw BESInternalError( oss.str(), __FILE__, __LINE__);
1997 }
1998
1999 try {
2000 dds_ptr->print(dds_file);
2001 }
2002 catch(...) {
2003 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2004 fclose(dds_file);
2005 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2006 }
2007
2008 fclose(dds_file);
2009 throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
2010 }
2011
2012 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2013 fclose(dds_file);
2014 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2015 }
2016
2017 fclose(dds_file);
2018 }
2019 return true;
2020
2021}
2022
2023// Write DAS to a binary cached file on disk.
2024bool HDF5RequestHandler::write_das_to_disk_cache(const string & das_cache_fname, DAS *das_ptr) {
2025
2026 BESDEBUG(HDF5_NAME, prolog << "Write DAS to disk cache " << das_cache_fname << endl);
2027 FILE *das_file = fopen(das_cache_fname.c_str(),"wb");
2028 if(nullptr == das_file) {
2029 string bes_error = "An error occurred trying to open a metadata cache file " + das_cache_fname;
2030 throw BESInternalError( bes_error, __FILE__, __LINE__);
2031 }
2032 else {
2033 int fd_md = fileno(das_file);
2034 struct flock *l_md;
2035 l_md = lock(F_WRLCK);
2036
2037 // hold a write(exclusive) lock to write metadata to a file.
2038 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
2039 fclose(das_file);
2040 ostringstream oss;
2041 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
2042 throw BESInternalError( oss.str(), __FILE__, __LINE__);
2043 }
2044
2045 try {
2046 write_das_to_file(das_ptr,das_file);
2047 }
2048 catch(...) {
2049 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2050 fclose(das_file);
2051 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2052 }
2053
2054 fclose(das_file);
2055 throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
2056 }
2057
2058 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2059 fclose(das_file);
2060 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2061 }
2062
2063 fclose(das_file);
2064
2065 }
2066
2067 return true;
2068
2069}
2070
2071// The wrapper function to call write_das_table_to_file to generate the cache.
2072void write_das_to_file(DAS*das_ptr,FILE* das_file) {
2073
2074 // When category_flag is 2, it marks the end of the file.
2075 uint8_t category_flag = 2;
2076 AttrTable* top_table = das_ptr->get_top_level_attributes();
2077 write_das_table_to_file(top_table,das_file);
2078
2079 // Add the final ending flag for retrieving the info.
2080 fwrite((const void*)&category_flag,1,1,das_file);
2081 return;
2082
2083}
2084
2085// The main function to write DAS to a file
2086void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
2087
2088 if(temp_table !=nullptr) {
2089
2090 // 2 is the end mark of an attribute table
2091 uint8_t category_flag = 2;
2092
2093 // Loop through the whole DAS top table
2094 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2095 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2096 AttrTable::Attr_iter top_it = top_startit;
2097 while(top_it !=top_endit) {
2098 AttrType atype = temp_table->get_attr_type(top_it);
2099 if(atype == Attr_unknown)
2100 throw InternalErr(__FILE__,__LINE__,"Unsupported DAS Attribute type");
2101 else if(atype!=Attr_container) {
2102 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr name is: "
2103 << temp_table->get_name(top_it) << endl);
2104 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr type is: "
2105 << temp_table->get_type(top_it) << endl);
2106 // For the debugging purpose
2107#if 0
2108 //unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2109 //cerr<<"Attribute values are "<<endl;
2110 //for (int i = 0; i <num_attrs;i++)
2111 // cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2112 //cerr<<endl;
2113 //write_das_attr_info(temp_table,top_it,das_file);
2114#endif
2115 // Write DAS attribute info to the file
2116 write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
2117 }
2118 else {
2119 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr container name is: "
2120 << (*top_it)->name << endl);
2121 // Write the container and then write the info. in this container
2122 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2123 write_container_name_to_file(sub_table->get_name(),das_file);
2124 write_das_table_to_file(sub_table,das_file);
2125
2126 // Write the end flag
2127 fwrite((const void*)&category_flag,1,1,das_file);
2128
2129 }
2130 ++top_it;
2131 }
2132
2133 }
2134}
2135
2136// Write container name to the disk file
2137void write_container_name_to_file(const string& cont_name,FILE *das_file) {
2138
2139 // 1 marks the starting of a container
2140 uint8_t category_flag = 1;
2141 vector<char> buf;
2142 size_t bytes_to_write = cont_name.size()+sizeof(size_t)+1;
2143 buf.resize(bytes_to_write);
2144 char*temp_pointer =buf.data();
2145 memcpy((void*)temp_pointer,(void*)&category_flag,1);
2146 temp_pointer++;
2147 temp_pointer=copy_str(temp_pointer,cont_name);
2148
2149 size_t bytes_to_be_written = fwrite((const void*)buf.data(),1,bytes_to_write,das_file);
2150 if(bytes_to_be_written != bytes_to_write)
2151 throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS container name to a cache");
2152 return;
2153}
2154
2155
2156// Write DAS attribute info. to the disk cache file
2157void write_das_attr_info(AttrTable* dtp,const string& attr_name, const string & attr_type,FILE * das_file) {
2158
2159 // 0 marks the starting of a DAS attribute
2160 uint8_t category_flag = 0;
2161
2162 unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
2163 vector<string> attr_values;
2164 size_t total_attr_values_size = 0;
2165 for (unsigned int i = 0; i <num_attr_elems;i++){
2166 attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
2167 total_attr_values_size += attr_values[i].size();
2168 }
2169 // Need to add a flag, value as 0 to indicate the attribute.
2170 // DAS: category flag, sizeof attirubte name, attribute name, size of attribute type, attribute type
2171 size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2* sizeof(size_t);
2172
2173 // One unsigned int to store the number of element elements i
2174 // + sizeof(size_t) * number of elements to store the number of characters for each attribute
2175 // (in DAP, every attribute is in string format)
2176 // +total size of all attribute values
2177 bytes_to_write_attr += sizeof(unsigned int) + num_attr_elems*sizeof(size_t)+total_attr_values_size;
2178
2179 vector<char>attr_buf;
2180 attr_buf.resize(bytes_to_write_attr);
2181 char* temp_attrp =attr_buf.data();
2182
2183 // The attribute flag
2184 memcpy((void*)temp_attrp,(void*)&category_flag,1);
2185 temp_attrp++;
2186
2187 // The attribute name and type
2188 temp_attrp=copy_str(temp_attrp,attr_name);
2189 temp_attrp=copy_str(temp_attrp,attr_type);
2190
2191 // Number of elements
2192 memcpy((void*)temp_attrp,(void*)&num_attr_elems,sizeof(unsigned int));
2193 temp_attrp+=sizeof(unsigned int);
2194
2195 // All attributes
2196 for (unsigned int i = 0; i <num_attr_elems;i++)
2197 temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
2198
2199 size_t bytes_to_be_written = fwrite((const void*)attr_buf.data(),1,bytes_to_write_attr,das_file);
2200 if(bytes_to_be_written != bytes_to_write_attr)
2201 throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS attribute to a cache");
2202
2203 return;
2204
2205}
2206
2207// Read DDS from a disk cache, this function is not used by default.
2208void HDF5RequestHandler::read_dds_from_disk_cache(BESDDSResponse* bdds, BESDataDDSResponse* data_bdds,
2209 bool build_data,const string & container_name,const string & h5_fname,
2210 const string & dds_cache_fname,const string &das_cache_fname, hid_t h5_fd,
2211 bool das_from_dc) {
2212
2213
2214 BESDEBUG(HDF5_NAME, prolog << "BEGIN dds_cache_fname: " << dds_cache_fname << endl);
2215
2216 DDS *dds;
2217 if(true == build_data)
2218 dds = data_bdds->get_dds();
2219 else
2220 dds = bdds->get_dds();
2221
2222 // write a function to pass the following with the lock.
2223 BaseTypeFactory tf;
2224 DDS tdds(&tf,name_path(h5_fname),"3.2");
2225 tdds.filename(h5_fname);
2226
2227 FILE *dds_file = fopen(dds_cache_fname.c_str(),"r");
2228 tdds.parse(dds_file);
2229 auto cache_dds = new DDS(tdds);
2230#if 0
2231cerr<<"before dds "<<endl;
2232dds->dump(cerr);
2233cerr<<"after dds "<<endl;
2234cerr<<"before tdds "<<endl;
2235cache_dds->dump(cerr);
2236cerr<<"after tdds "<<endl;
2237#endif
2238 if(dds != nullptr)
2239 delete dds;
2240
2241 Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
2242
2243 add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
2244 if(true == build_data)
2245 data_bdds->set_dds(cache_dds);
2246 else
2247 bdds->set_dds(cache_dds);
2248 if(dds_file !=nullptr)
2249 fclose(dds_file);
2250
2251 if (dds_cache) {
2252 // add a copy
2253 BESDEBUG(HDF5_NAME, prolog << "For memory cache, DDS added to the cache for : " << h5_fname << endl);
2254 dds_cache->add(new DDS(*cache_dds), h5_fname);
2255 }
2256
2257}
2258
2259// Add DAS to DDS.
2260void HDF5RequestHandler::add_das_to_dds(DDS *dds, const string &/*container_name*/, const string &filename,
2261 const string &das_cache_fname, hid_t h5_fd, bool das_from_dc) {
2262
2263 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
2264
2265 // Check DAS memory cache
2266 DAS *das = nullptr ;
2267 bool use_das_cache = false;
2268 if (das_cache)
2269 das = static_cast<DAS*>(das_cache->get(filename));
2270 if (das)
2271 use_das_cache = true;
2272
2273 if (true == use_das_cache) {
2274 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2275 dds->transfer_attributes(das); // no need to copy the cached DAS
2276 }
2277
2278 else {
2279
2280 das = new DAS ;
2281
2282 // The following block is commented out because the attribute containers in DDX disappear
2283 // when the container_name of DAS is added. Without adding the container_name of DAS,
2284 // the attribute containers show up in DDX. This information is re-discovered while working on
2285 // https://bugs.earthdata.nasa.gov/browse/HYRAX-714 although the following code was commented
2286 // out long time ago. KY 2022-05-27
2287#if 0
2288 if (!container_name.empty())
2289 das->container_name(container_name);
2290#endif
2291 if(das_from_dc == true)
2292 read_das_from_disk_cache(das_cache_fname,das);
2293 else {
2294 // This bool is for the case, when DDS is read from a cache then we need to open the HDF5 file.
2295 bool h5_file_open = true;
2296 if(h5_fd == -1)
2297 h5_file_open = false;
2298 if (true == _usecf) {
2299 // go to the CF option
2300 if(h5_file_open == false)
2301 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2302
2303 read_cfdas( *das,filename,h5_fd);
2304 if(h5_file_open == false)
2305 H5Fclose(h5_fd);
2306 }
2307 else {
2308 if(h5_file_open == false)
2309 h5_fd = get_fileid(filename.c_str());
2310 find_gloattr(h5_fd, *das);
2311 depth_first(h5_fd, "/", *das);
2312 if(h5_file_open == false)
2313 close_fileid(h5_fd);
2314 }
2315
2316 Ancillary::read_ancillary_das( *das, filename ) ;
2317
2318 if(das_cache_fname!="" && das_from_dc == false)
2319 write_das_to_disk_cache(das_cache_fname,das);
2320 }
2321
2322 dds->transfer_attributes(das);
2323
2324 if (das_cache) {
2325 // add a copy
2326 BESDEBUG(HDF5_NAME, prolog << "For memory cache, DAS added to the cache for : " << filename << endl);
2327 das_cache->add(new DAS(*das), filename);
2328 }
2329 delete das;
2330
2331 }
2332
2333}
2334
2335bool obtain_beskeys_info(const string& key, bool & has_key) {
2336
2337 bool ret_value = false;
2338 string doset ="";
2339 TheBESKeys::TheKeys()->get_value( key, doset, has_key ) ;
2340 if(has_key) {
2341 const string dosettrue ="true";
2342 const string dosetyes = "yes";
2343 doset = BESUtil::lowercase(doset) ;
2344 ret_value = (dosettrue == doset || dosetyes == doset);
2345 }
2346 return ret_value;
2347}
2348
2349#if 0
2350bool is_beskeys_set_true(const string& bes_value) {
2351
2352 const string dosettrue ="true";
2353 const string dosetyes = "yes";
2354 string doset = BESUtil::lowercase(bes_value) ;
2355 return (dosettrue == doset || dosetyes == doset);
2356
2357}
2358bool check_and_set_beskeys(const string key) {
2359
2360 bool found = false;
2361 string doset ="";
2362 const string dosettrue ="true";
2363 const string dosetyes = "yes";
2364
2365 TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2366 if( true == found ) {
2367 doset = BESUtil::lowercase( doset ) ;
2368 }
2369 BESDEBUG(HDF5_NAME, prolog << "Key: " << key << (found?(" was found. value: "+doset):" was not found.") << endl);
2370 return found && (dosettrue == doset || dosetyes == doset);
2371
2372}
2373#endif
2374
2375// get_uint_key and get_float_key are copied from the netCDF handler.
2376
2377static unsigned int get_uint_key(const string &key, unsigned int def_val)
2378{
2379 bool found = false;
2380 string doset = "";
2381
2382 TheBESKeys::TheKeys()->get_value(key, doset, found);
2383 if (true == found) {
2384 // In C++11, stoi is better.
2385 return atoi(doset.c_str()); // use better code TODO
2386 }
2387 else {
2388 return def_val;
2389 }
2390}
2391
2392static unsigned long get_ulong_key(const string &key, unsigned long def_val)
2393{
2394 bool found = false;
2395 string doset = "";
2396
2397 TheBESKeys::TheKeys()->get_value(key, doset, found);
2398 if (true == found) {
2399 // In C++11, stoull is better.
2400 return atol(doset.c_str()); // use better code TODO
2401 }
2402 else {
2403 return def_val;
2404 }
2405}
2406static float get_float_key(const string &key, float def_val)
2407{
2408 bool found = false;
2409 string doset = "";
2410
2411 TheBESKeys::TheKeys()->get_value(key, doset, found);
2412 if (true == found) {
2413 return atof(doset.c_str()); // use better code TODO
2414 }
2415 else {
2416 return def_val;
2417 }
2418}
2419
2420static string get_beskeys(const string &key) {
2421
2422 bool found = false;
2423 string ret_value ="";
2424
2425 TheBESKeys::TheKeys()->get_value( key, ret_value, found ) ;
2426 return ret_value;
2427
2428}
2429
2430// The function to copy a string to a memory buffer.
2431char* copy_str(char*temp_ptr,const string & str) {
2432
2433 size_t str_size=str.size();
2434 memcpy((void*)temp_ptr,(void*)&str_size,sizeof(size_t));
2435 temp_ptr+=sizeof(size_t);
2436 vector<char>temp_vc2(str.begin(),str.end());
2437 memcpy((void*)temp_ptr,(void*)temp_vc2.data(),str.size());
2438 temp_ptr+=str.size();
2439 return temp_ptr;
2440
2441}
2442
2443
2444// Obtain the string from a memory buffer.
2445// Note: both char* and string(as a reference) will be returned
2446// The attribute binary first stores the size of the string, then the string itself
2447char* obtain_str(char*temp_ptr,string & str) {
2448
2449 size_t oname_size = *((size_t *)temp_ptr);
2450 temp_ptr = temp_ptr + sizeof(size_t);
2451 string oname;
2452 for(unsigned int i =0; i<oname_size; i++){
2453 oname.push_back(*temp_ptr);
2454 ++temp_ptr;
2455 }
2456 str = oname;
2457 return temp_ptr;
2458
2459}
2460
2461// For our case, there are no global attributes for DAS.
2462// The global attribures are always under HDF_GLOBAL.
2463// The main function to obtain the DAS info. from the cache.
2464char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at_par) {
2465
2466 // 3 is only for the code to come into the loop.
2467 uint8_t flag =3;
2468 while(flag !=2) {
2469 flag = *((uint8_t*)(temp_pointer));
2470 BESDEBUG(HDF5_NAME, prolog << "Build DAS from the disk cache file flag: "
2471 <<" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2472 <<" flag = 3; the initial value to get the attribute retrieval process started."
2473 <<" The flag value is "
2474 << (int)flag <<endl);
2475 temp_pointer++;
2476
2477 if(flag ==1) {
2478 string container_name;
2479 temp_pointer = obtain_str(temp_pointer,container_name);
2480 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, container name is " << container_name << endl);
2481
2482 // Remember the current Attribute table state
2483 AttrTable*temp_at_par = at_par;
2484 if(at_par == nullptr)
2485 at_par = das->add_table(container_name, new AttrTable);
2486 else
2487 at_par = at_par->append_container(container_name);
2488
2489 temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2490 // MUST resume the original state
2491 at_par = temp_at_par;
2492
2493 }
2494 else if(flag == 0) {
2495 // The attribute must have a table.
2496 if(at_par ==nullptr)
2497 throw BESInternalError( "The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2498
2499 // Attribute name
2500 string attr_name;
2501 temp_pointer = obtain_str(temp_pointer,attr_name);
2502 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr name is: " << attr_name << endl);
2503
2504 // Attribute type
2505 string attr_type;
2506 temp_pointer = obtain_str(temp_pointer,attr_type);
2507 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr type is: " << attr_type << endl);
2508
2509 // Attribute values
2510 unsigned int num_values = *((unsigned int*)(temp_pointer));
2511 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, number of attribute values is: " << num_values << endl);
2512 temp_pointer+=sizeof(unsigned int);
2513
2514 vector <string> attr_values;
2515
2516 for(unsigned int i = 0; i<num_values; i++) {
2517 string attr_value;
2518 temp_pointer = obtain_str(temp_pointer,attr_value);
2519 attr_values.push_back(attr_value);
2520 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attribute value is: " << attr_value << endl);
2521 }
2522
2523 at_par->append_attr(attr_name,attr_type,&attr_values);
2524 }
2525
2526 }
2527 return temp_pointer;
2528
2529}
2530
2531// The debugging function to get attribute info.
2532void get_attr_contents(AttrTable*temp_table) {
2533 if(temp_table !=nullptr) {
2534 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2535 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2536 AttrTable::Attr_iter top_it = top_startit;
2537 while(top_it !=top_endit) {
2538 AttrType atype = temp_table->get_attr_type(top_it);
2539 if(atype == Attr_unknown)
2540 cerr<<"unsupported DAS attributes" <<endl;
2541 else if(atype!=Attr_container) {
2542
2543 cerr<<"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2544 cerr<<"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2545 unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2546 cerr<<"Attribute values are "<<endl;
2547 for (unsigned int i = 0; i <num_attrs;i++)
2548 cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2549 cerr<<endl;
2550 }
2551 else {
2552 cerr<<"Coming to the attribute container. "<<endl;
2553 cerr<<"container name is "<<(*top_it)->name <<endl;
2554 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2555 cerr<<"container table name is "<<sub_table->get_name() <<endl;
2556 get_attr_contents(sub_table);
2557 }
2558 ++top_it;
2559 }
2560
2561 }
2562}
2563
2564
2565void HDF5RequestHandler::add_attributes(BESDataHandlerInterface &dhi) {
2566
2567
2568 BESResponseObject *response = dhi.response_handler->get_response_object();
2569 BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
2570 if (!bdds)
2571 throw BESInternalError("cast error", __FILE__, __LINE__);
2572 DDS *dds = bdds->get_dds();
2573 string container_name = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name(): "";
2574 string filename = dhi.container->access();
2575 DAS* das = nullptr;
2576 bool das_from_mcache = false;
2577 if(das_cache) {
2578 das = static_cast<DAS*>(das_cache->get(filename));
2579 if(das) {
2580 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2581 dds->transfer_attributes(das); // no need to copy the cached DAS
2582 das_from_mcache = true;
2583 }
2584 }
2585
2586 if(false == das_from_mcache) {
2587 das = new DAS;
2588 // This looks at the 'use explicit containers' prop, and if true
2589 // sets the current container for the DAS.
2590 if (!container_name.empty()) das->container_name(container_name);
2591
2592 hid_t h5_fd =-1;
2593 if (true == _usecf) {
2594 // go to the CF option
2595 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2596
2597 read_cfdas( *das,filename,h5_fd);
2598
2599 H5Fclose(h5_fd);
2600 }
2601 else {
2602 h5_fd = get_fileid(filename.c_str());
2603 find_gloattr(h5_fd, *das);
2604 depth_first(h5_fd, "/", *das);
2605 close_fileid(h5_fd);
2606 }
2607
2608
2609 Ancillary::read_ancillary_das(*das, filename);
2610
2611 dds->transfer_attributes(das);
2612
2613 // Only free the DAS if it's not added to the cache
2614 if (das_cache) {
2615 // add a copy
2616 BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
2617 das_cache->add(das, filename);
2618 }
2619 else {
2620 delete das;
2621 }
2622 }
2623 BESDEBUG(HDF5_NAME, prolog << "Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2624 bdds->set_ia_flag(true);
2625 return;
2626
2627}
2628
2629
include the entry functions to execute the handlers
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
BESContainer * container
pointer to current container in this interface
Base exception class for the BES with basic string message.
Definition: BESError.h:59
std::string get_message() const
get the error message for this exception
Definition: BESError.h:111
informational response object
Definition: BESInfo.h:63
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:254
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:617
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
Helper functions for generating DAS attributes and a function to check BES Key.
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void find_gloattr(hid_t file, DAS &das)
Definition: h5das.cc:482
bool breadth_first(const hid_t file_id, hid_t pid, const char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
Definition: h5dmr.cc:321
hid_t get_fileid(const char *filename)
Definition: h5get.cc:412
void close_fileid(hid_t fid)
Definition: h5get.cc:434
The main header of the HDF5 OPeNDAP handler.