37#include <libdap/DMR.h>
38#include <libdap/D4BaseTypeFactory.h>
39#include <BESDMRResponse.h>
40#include <libdap/mime_util.h>
41#include <libdap/InternalErr.h>
42#include <libdap/Ancillary.h>
43#include <libdap/debug.h>
45#include "HDF4RequestHandler.h"
46#include <BESResponseNames.h>
47#include <BESDapNames.h>
48#include <BESDASResponse.h>
49#include <BESDDSResponse.h>
50#include <BESDataDDSResponse.h>
52#include <BESResponseHandler.h>
53#include <BESVersionInfo.h>
54#include <BESServiceRegistry.h>
56#include <TheBESKeys.h>
57#include <libdap/InternalErr.h>
58#include <BESInternalError.h>
59#include <BESDapError.h>
60#include <BESStopWatch.h>
62#include "BESDataNames.h"
64#include <libdap/Ancillary.h>
65#include "config_hdf.h"
74#include "HDFFloat32.h"
75#include "HDFSPArray_RealField.h"
85bool check_beskeys(
const string &);
86bool get_beskeys(
const string &,
string &);
88extern void read_das(DAS & das,
const string & filename);
89extern void read_dds(DDS & dds,
const string & filename);
91extern bool read_dds_hdfsp(DDS & dds,
const string & filename,int32 sdfd, int32 fileid,
HDFSP::File*h4file);
93extern bool read_das_hdfsp(DAS & das,
const string & filename,int32 sdfd, int32 fileid,
HDFSP::File**h4fileptr);
95extern void read_das_sds(DAS & das,
const string & filename,int32 sdfd,
bool ecs_metadata,
HDFSP::File**h4fileptr);
96extern void read_dds_sds(DDS &dds,
const string & filename,int32 sdfd,
HDFSP::File*h4file,
bool dds_set_cache);
100void read_das_use_eos2lib(DAS & das,
const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,
bool ecs_metadata,
HDFSP::File**h4file,HDFEOS2::File**eosfile);
101void read_dds_use_eos2lib(DDS & dds,
const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,
HDFSP::File*h4file,HDFEOS2::File*eosfile);
102void close_fileid(
const int sdfd,
const int fileid,
const int gridfd,
const int swathfd,
HDFSP::File*h4file,HDFEOS2::File*eosfile);
106void close_hdf4_fileid(
const int sdfd,
const int fileid,
HDFSP::File*h4file);
107bool rw_das_cache_file(
const string & filename, DAS *das_ptr,
bool rw_flag);
108bool r_dds_cache_file(
const string & cache_filename, DDS *dds_ptr,
const string & hdf4_filename);
111bool HDF4RequestHandler::_usecf =
false;
114bool HDF4RequestHandler::_pass_fileid =
false;
115bool HDF4RequestHandler::_disable_structmeta =
false;
116bool HDF4RequestHandler::_enable_special_eos =
false;
117bool HDF4RequestHandler::_disable_scaleoffset_comp =
false;
118bool HDF4RequestHandler::_disable_ecsmetadata_min =
false;
119bool HDF4RequestHandler::_disable_ecsmetadata_all =
false;
123bool HDF4RequestHandler::_enable_eosgeo_cachefile =
false;
124bool HDF4RequestHandler::_enable_data_cachefile =
false;
125bool HDF4RequestHandler::_enable_metadata_cachefile=
false;
128bool HDF4RequestHandler::_enable_hybrid_vdata =
false;
129bool HDF4RequestHandler::_enable_ceres_vdata =
false;
130bool HDF4RequestHandler::_enable_vdata_attr =
false;
131bool HDF4RequestHandler::_enable_vdata_desc_attr =
false;
132bool HDF4RequestHandler::_disable_vdata_nameclashing_check =
false;
133bool HDF4RequestHandler::_enable_vgroup_attr =
false;
136bool HDF4RequestHandler::_enable_check_modis_geo_file =
false;
137bool HDF4RequestHandler::_enable_swath_grid_attr =
false;
138bool HDF4RequestHandler::_enable_ceres_merra_short_name =
false;
139bool HDF4RequestHandler::_enable_check_scale_offset_type =
false;
140bool HDF4RequestHandler::_disable_swath_dim_map =
false;
143bool HDF4RequestHandler::_cache_latlon_path_exist =
false;
144string HDF4RequestHandler::_cache_latlon_path =
"";
145bool HDF4RequestHandler::_cache_latlon_prefix_exist =
false;
146string HDF4RequestHandler::_cache_latlon_prefix =
"";
147bool HDF4RequestHandler::_cache_latlon_size_exist =
false;
148long HDF4RequestHandler::_cache_latlon_size =0;
149bool HDF4RequestHandler::_cache_metadata_path_exist =
false;
150string HDF4RequestHandler::_cache_metadata_path =
"";
152HDF4RequestHandler::HDF4RequestHandler(
const string & name) :
154 add_handler(DAS_RESPONSE, HDF4RequestHandler::hdf4_build_das);
155 add_handler(DDS_RESPONSE, HDF4RequestHandler::hdf4_build_dds);
156 add_handler(DATA_RESPONSE, HDF4RequestHandler::hdf4_build_data);
157 add_handler(DMR_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
158 add_handler(DAP4DATA_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
159 add_handler(HELP_RESPONSE, HDF4RequestHandler::hdf4_build_help);
160 add_handler(VERS_RESPONSE, HDF4RequestHandler::hdf4_build_version);
162 _usecf = check_beskeys(
"H4.EnableCF");
166 _pass_fileid = check_beskeys(
"H4.EnablePassFileID");
167 _disable_structmeta = check_beskeys(
"H4.DisableStructMetaAttr");
168 _enable_special_eos = check_beskeys(
"H4.EnableSpecialEOS");
169 _disable_scaleoffset_comp = check_beskeys(
"H4.DisableScaleOffsetComp");
170 _disable_ecsmetadata_min = check_beskeys(
"H4.DisableECSMetaDataMin");
171 _disable_ecsmetadata_all = check_beskeys(
"H4.DisableECSMetaDataAll");
175 _enable_eosgeo_cachefile = check_beskeys(
"H4.EnableEOSGeoCacheFile");
176 _enable_data_cachefile = check_beskeys(
"H4.EnableDataCacheFile");
177 _enable_metadata_cachefile = check_beskeys(
"H4.EnableMetaDataCacheFile");
180 _enable_hybrid_vdata = check_beskeys(
"H4.EnableHybridVdata");
181 _enable_ceres_vdata = check_beskeys(
"H4.EnableCERESVdata");
182 _enable_vdata_attr = check_beskeys(
"H4.EnableVdata_to_Attr");
183 _enable_vdata_desc_attr = check_beskeys(
"H4.EnableVdataDescAttr");
184 _disable_vdata_nameclashing_check = check_beskeys(
"H4.DisableVdataNameclashingCheck");
185 _enable_vgroup_attr = check_beskeys(
"H4.EnableVgroupAttr");
188 _enable_check_modis_geo_file = check_beskeys(
"H4.EnableCheckMODISGeoFile");
189 _enable_swath_grid_attr = check_beskeys(
"H4.EnableSwathGridAttr");
190 _enable_ceres_merra_short_name = check_beskeys(
"H4.EnableCERESMERRAShortName");
191 _enable_check_scale_offset_type = check_beskeys(
"H4.EnableCheckScaleOffsetType");
193 _disable_swath_dim_map = check_beskeys(
"H4.DisableSwathDimMap");
196 _cache_latlon_path_exist =get_beskeys(
"HDF4.Cache.latlon.path",_cache_latlon_path);
197 _cache_latlon_prefix_exist =get_beskeys(
"HDF4.Cache.latlon.prefix",_cache_latlon_prefix);
198 string temp_cache_latlon_size;
199 _cache_latlon_size_exist =get_beskeys(
"HDF4.Cache.latlon.size",temp_cache_latlon_size);
200 if(_cache_latlon_size_exist ==
true) {
201 istringstream iss(temp_cache_latlon_size);
202 iss >> _cache_latlon_size;
205 _cache_metadata_path_exist =get_beskeys(
"H4.Cache.metadata.path",_cache_metadata_path);
210HDF4RequestHandler::~HDF4RequestHandler() {
219 sw.
start(
"HDF4RequestHandler::hdf4_build_das", dhi.
data[REQUEST_ID]);
237 if((base_file_name.size() >12) && (base_file_name.compare(0,4,
"AIRS") == 0)
238 && ((base_file_name.find(
".L3.")!=string::npos) || (base_file_name.find(
".L2.")!=string::npos))
239 && (base_file_name.find(
".v6.")!=string::npos)) {
240 return hdf4_build_das_cf_sds(dhi);
252 DAS *das = bdas->get_das();
258 if (
true == _usecf) {
267 sdfd = SDstart (accessed.c_str(), DFACC_READ);
269 string invalid_file_msg=
"HDF4 SDstart error for the file ";
270 invalid_file_msg +=accessed;
271 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
276 fileid = Hopen(accessed.c_str(), DFACC_READ,0);
279 string invalid_file_msg=
"HDF4 Hopen error for the file ";
280 invalid_file_msg +=accessed;
281 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
285#ifdef USE_HDFEOS2_LIB
289 HDFEOS2::File *eosfile =
nullptr;
293 gridfd = GDopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
297 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
298 invalid_file_msg +=accessed;
299 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
304 swathfd = SWopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
309 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
310 invalid_file_msg +=accessed;
311 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
316 bool ecs_metadata = !_disable_ecsmetadata_all;
318if(ecs_metadata ==
true)
319cerr<<
"output ecs metadata "<<endl;
321cerr<<
"Don't output ecs metadata "<<endl;
323 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
326 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
332 if(eosfile !=
nullptr)
339 read_das_hdfsp(*das,accessed,sdfd,fileid,&h4file);
342 close_hdf4_fileid(sdfd,fileid,h4file);
347 close_hdf4_fileid(sdfd,fileid,h4file);
350 read_das(*das,accessed);
352 Ancillary::read_ancillary_das(*das, accessed);
353 bdas->clear_container();
359 catch (InternalErr & e) {
360 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
364 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
368 string s =
"unknown exception caught building HDF4 DAS";
369 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
381 sw.
start(
"HDF4RequestHandler::hdf4_build_das", dhi.
data[REQUEST_ID]);
399 if((base_file_name.size() >12) && (base_file_name.compare(0,4,
"AIRS") == 0)
400 && ((base_file_name.find(
".L3.")!=string::npos) || (base_file_name.find(
".L2.")!=string::npos))
401 && (base_file_name.find(
".v6.")!=string::npos)) {
402 return hdf4_build_dds_cf_sds(dhi);
409struct timeval start_time,end_time;
410gettimeofday(&start_time,
nullptr);
420 DDS *dds = bdds->get_dds();
426 dds->filename(accessed);
432 if (
true == _usecf) {
440 sdfd = SDstart (accessed.c_str(), DFACC_READ);
442 string invalid_file_msg=
"HDF4 SDstart error for the file ";
443 invalid_file_msg +=accessed;
444 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
449 fileid = Hopen(accessed.c_str(), DFACC_READ,0);
452 string invalid_file_msg=
"HDF4 Hopen error for the file ";
453 invalid_file_msg +=accessed;
454 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
458#ifdef USE_HDFEOS2_LIB
463 HDFEOS2::File *eosfile =
nullptr;
467 gridfd = GDopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
471 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
472 invalid_file_msg +=accessed;
473 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
478 swathfd = SWopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
483 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
484 invalid_file_msg +=accessed;
485 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
490 bool ecs_metadata = !_disable_ecsmetadata_all;
491 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
492 Ancillary::read_ancillary_das(*das, accessed);
495 read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
498 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
502 if(eosfile !=
nullptr)
510 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
511 Ancillary::read_ancillary_das(*das, accessed);
514 read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
517 close_hdf4_fileid(sdfd,fileid,h4file);
522 close_hdf4_fileid(sdfd,fileid,h4file);
525 read_das(*das, accessed);
526 Ancillary::read_ancillary_das(*das, accessed);
527 read_dds(*dds, accessed);
532gettimeofday(&end_time,
nullptr);
533int total_time_spent = (end_time.tv_sec - start_time.tv_sec)*1000000 +end_time.tv_usec-start_time.tv_usec;
534cerr<<
"total time spent for DDS buld is "<<total_time_spent<<
"micro seconds "<<endl;
537 Ancillary::read_ancillary_dds(*dds, accessed);
539 dds->transfer_attributes(das);
541 bdds->set_constraint(dhi);
543 bdds->clear_container();
548 catch (InternalErr & e) {
549 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
553 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
557 string s =
"unknown exception caught building HDF4 DDS";
558 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
569 sw.
start(
"HDF4RequestHandler::hdf4_build_data", dhi.
data[REQUEST_ID]);
592 if((base_file_name.size() >12) && (base_file_name.compare(0,4,
"AIRS") == 0)
593 && ((base_file_name.find(
".L3.")!=string::npos) || (base_file_name.find(
".L2.")!=string::npos))
594 && (base_file_name.find(
".v6.")!=string::npos)) {
596 BESDEBUG(
"h4",
"Coming to read the data of AIRS level 3 or level 2 products." << endl);
598 if(
true == _pass_fileid)
599 return hdf4_build_data_cf_sds_with_IDs(dhi);
601 return hdf4_build_data_cf_sds(dhi);
605 if(
true == _pass_fileid)
606 return hdf4_build_data_with_IDs(dhi);
619 DDS *dds = bdds->get_dds();
625 dds->filename(accessed);
631 if (
true == _usecf) {
637 sdfd = SDstart (accessed.c_str(), DFACC_READ);
639 string invalid_file_msg=
"HDF4 SDstart error for the file ";
640 invalid_file_msg +=accessed;
641 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
646 fileid = Hopen(accessed.c_str(), DFACC_READ,0);
649 string invalid_file_msg=
"HDF4 Hopen error for the file ";
650 invalid_file_msg +=accessed;
651 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
656#ifdef USE_HDFEOS2_LIB
660 HDFEOS2::File *eosfile =
nullptr;
664 gridfd = GDopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
668 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
669 invalid_file_msg +=accessed;
670 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
675 swathfd = SWopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
680 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
681 invalid_file_msg +=accessed;
682 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
692 bool ecs_metadata =
true;
694 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
695 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
696 ecs_metadata =
false;
698 if((
true == _disable_ecsmetadata_min)
699 || (
true == _disable_ecsmetadata_all))
700 ecs_metadata =
false;
702 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
703 Ancillary::read_ancillary_das(*das, accessed);
706 read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
709 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
713 if(eosfile !=
nullptr)
720 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
721 Ancillary::read_ancillary_das(*das, accessed);
724 read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
727 close_hdf4_fileid(sdfd,fileid,h4file);
731 close_hdf4_fileid(sdfd,fileid,h4file);
734 read_das(*das, accessed);
735 Ancillary::read_ancillary_das(*das, accessed);
736 read_dds(*dds, accessed);
739 Ancillary::read_ancillary_dds(*dds, accessed);
740 dds->transfer_attributes(das);
741 bdds->set_constraint(dhi);
742 bdds->clear_container();
749 catch (InternalErr & e) {
750 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
754 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
758 string s =
"unknown exception caught building DAP2 Data Response from an HDF4 data resource";
759 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
769 sw.
start(
"HDF4RequestHandler::hdf4_build_data_with_IDs", dhi.
data[REQUEST_ID]);
774#ifdef USE_HDFEOS2_LIB
777 HDFEOS2::File *eosfile =
nullptr;
793 delete bdds->get_dds();
802 hdds->filename(accessed);
811 sdfd = SDstart (accessed.c_str(), DFACC_READ);
813 string invalid_file_msg=
"HDF4 SDstart error for the file ";
814 invalid_file_msg +=accessed;
815 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
819 fileid = Hopen(accessed.c_str(), DFACC_READ,0);
822 string invalid_file_msg=
"HDF4 Hopen error for the file ";
823 invalid_file_msg +=accessed;
824 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
828#ifdef USE_HDFEOS2_LIB
834 gridfd = GDopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
838 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
839 invalid_file_msg +=accessed;
840 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
845 swathfd = SWopen(
const_cast < char *
>(accessed.c_str()), DFACC_READ);
850 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
851 invalid_file_msg +=accessed;
852 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
856 hdds->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
861 bool ecs_metadata =
true;
863 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
864 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
866 if((
true == _disable_ecsmetadata_min)
867 || (
true == _disable_ecsmetadata_all))
868 ecs_metadata =
false;
870 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
876 Ancillary::read_ancillary_das(*das, accessed);
879 read_dds_use_eos2lib(*hdds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
881 if(eosfile !=
nullptr)
885 hdds->setHDF4Dataset(sdfd,fileid);
886 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
887 Ancillary::read_ancillary_das(*das, accessed);
890 read_dds_hdfsp(*hdds, accessed,sdfd,fileid,h4file);
892 if(h4file !=
nullptr)
895 Ancillary::read_ancillary_dds(*hdds, accessed);
897 hdds->transfer_attributes(das);
899 bdds->set_constraint(dhi);
901 bdds->clear_container();
907#ifdef USE_HDFEOS2_LIB
918 catch (InternalErr & e) {
919 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
923 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
927#ifdef USE_HDFEOS2_LIB
928 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
930 close_hdf4_fileid(sdfd,fileid,h4file);
932 string s =
"unknown exception caught building DAP2 data response from an HDF4 data resource";
933 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
955 DDS *dds = bdds->get_dds();
963 dds->filename(accessed);
970 bool das_set_cache =
false;
971 bool dds_set_cache =
false;
972 bool dds_das_get_cache =
false;
978 string check_enable_mcache_key=
"H4.EnableMetaDataCacheFile";
979 bool turn_on_enable_mcache_key=
false;
980 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
981 if(
true == turn_on_enable_mcache_key) {
983 if(
true == _enable_metadata_cachefile) {
989 string key =
"H4.Cache.metadata.path";
992 if(
true == _cache_metadata_path_exist) {
995 das_filename = _cache_metadata_path +
"/" + base_file_name +
"_das";
996 dds_filename = _cache_metadata_path +
"/" + base_file_name +
"_dds";
1000 das_set_cache = rw_das_cache_file(das_filename,das,
false);
1004 dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1009 if((
false == das_set_cache)&&(
false == dds_set_cache))
1010 dds_das_get_cache =
true;
1016 if(
false == dds_das_get_cache) {
1019 sdfd = SDstart (accessed.c_str(), DFACC_READ);
1022 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1023 invalid_file_msg +=accessed;
1024 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1031 bool ecs_metadata = !_disable_ecsmetadata_all;
1033 bool ecs_metadata =
true;
1034 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1035 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1036 ecs_metadata =
false;
1039 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1041 Ancillary::read_ancillary_das(*das, accessed);
1045 read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1048 if(
true == das_set_cache) {
1049 if(das_filename ==
"")
1050 throw InternalErr(__FILE__,__LINE__,
"DAS cache file name should be set ");
1051 rw_das_cache_file(das_filename,das,
true);
1056 Ancillary::read_ancillary_dds(*dds, accessed);
1057 dds->transfer_attributes(das);
1058 bdds->set_constraint(dhi);
1060 bdds->clear_container();
1062 if(h4file !=
nullptr)
1078 catch (InternalErr & e) {
1079 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1080 __FILE__, __LINE__);
1083 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1084 __FILE__, __LINE__);
1089 if(h4file !=
nullptr)
1091 string s =
"unknown exception caught building HDF4 DataDDS";
1092 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1114 auto das = bdas->get_das();
1120 bool das_set_cache =
false;
1121 bool das_get_cache =
false;
1122 string das_filename;
1124 string check_enable_mcache_key=
"H4.EnableMetaDataCacheFile";
1125 bool turn_on_enable_mcache_key=
false;
1126 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1127 if(
true == turn_on_enable_mcache_key) {
1129 if(
true == _enable_metadata_cachefile) {
1131 string md_cache_dir;
1132 string key =
"H4.Cache.metadata.path";
1137 if(
true == _cache_metadata_path_exist) {
1140 das_filename = _cache_metadata_path +
"/" + base_file_name +
"_das";
1144 das_set_cache = rw_das_cache_file(das_filename,das,
false);
1147 if(
false == das_set_cache)
1148 das_get_cache =
true;
1153 if(
false == das_get_cache) {
1155 sdfd = SDstart (accessed.c_str(), DFACC_READ);
1157 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1158 invalid_file_msg +=accessed;
1159 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1165 bool ecs_metadata =
true;
1166 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1167 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1168 ecs_metadata =
false;
1172 bool ecs_metadata = !_disable_ecsmetadata_all;
1174 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1175 Ancillary::read_ancillary_das(*das, accessed);
1178 if(
true == das_set_cache)
1179 rw_das_cache_file(das_filename,das,
true);
1183 bdas->clear_container();
1185 if(h4file !=
nullptr)
1196 catch (InternalErr & e) {
1197 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1198 __FILE__, __LINE__);
1201 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1202 __FILE__, __LINE__);
1207 if(h4file !=
nullptr)
1209 string s =
"unknown exception caught building HDF4 DataDDS";
1210 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1230 DDS *dds = bdds->get_dds();
1233 dds->filename(accessed);
1246 bool das_set_cache =
false;
1247 bool dds_set_cache =
false;
1248 bool dds_das_get_cache =
false;
1249 string das_filename;
1250 string dds_filename;
1253 string check_enable_mcache_key=
"H4.EnableMetaDataCacheFile";
1254 bool _disable_ecsmetadata_allturn_on_enable_mcache_key=
false;
1255 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1256 if(
true == turn_on_enable_mcache_key) {
1258 if(
true == _enable_metadata_cachefile) {
1260 if(
true == _cache_metadata_path_exist) {
1261 BESDEBUG(
"h4",
"H4.Cache.metadata.path key is set and metadata cache key is set." << endl);
1266 das_filename = _cache_metadata_path +
"/" + base_file_name +
"_das_dd";
1267 dds_filename = _cache_metadata_path +
"/" + base_file_name +
"_dds";
1270 das_set_cache = rw_das_cache_file(das_filename,das,
false);
1273 dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1275 if((
false == das_set_cache)&&(
false == dds_set_cache))
1276 dds_das_get_cache =
true;
1279 if(
false == dds_das_get_cache) {
1282 sdfd = SDstart (accessed.c_str(), DFACC_READ);
1284 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1285 invalid_file_msg +=accessed;
1286 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1294 bool ecs_metadata =
true;
1296 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1297 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1299 if((
true == _disable_ecsmetadata_min)
1300 || (
true == _disable_ecsmetadata_all))
1301 ecs_metadata =
false;
1303 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1305 Ancillary::read_ancillary_das(*das, accessed);
1308 if(
true == das_set_cache) {
1309 rw_das_cache_file(das_filename,das,
true);
1314 read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1318 Ancillary::read_ancillary_dds(*dds, accessed);
1320 dds->transfer_attributes(das);
1322 bdds->set_constraint(dhi);
1324 bdds->clear_container();
1326 if(h4file !=
nullptr)
1336 catch (InternalErr & e) {
1337 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1338 __FILE__, __LINE__);
1341 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1342 __FILE__, __LINE__);
1347 if(h4file !=
nullptr)
1349 string s =
"unknown exception caught building HDF4 DataDDS";
1350 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1376 auto hdds =
new HDF4DDS(bdds->get_dds());
1379 delete bdds->get_dds();
1381 bdds->set_dds(hdds);
1385 hdds->filename(accessed);
1392 sdfd = SDstart (accessed.c_str(), DFACC_READ);
1394 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1395 invalid_file_msg +=accessed;
1396 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1400 hdds->setHDF4Dataset(sdfd,-1);
1405 bool ecs_metadata =
true;
1407 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1408 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1410 if((
true == _disable_ecsmetadata_min)
1411 || (
true == _disable_ecsmetadata_all))
1412 ecs_metadata =
false;
1414 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1419 Ancillary::read_ancillary_das(*das, accessed);
1422 read_dds_sds(*hdds, accessed,sdfd,h4file,
false);
1424 if(h4file !=
nullptr)
1427 Ancillary::read_ancillary_dds(*hdds, accessed);
1429 hdds->transfer_attributes(das);
1431 bdds->set_constraint(dhi);
1433 bdds->clear_container();
1440 catch (InternalErr & e) {
1441 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1442 __FILE__, __LINE__);
1445 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1446 __FILE__, __LINE__);
1451 if(h4file !=
nullptr)
1453 string s =
"unknown exception caught building HDF4 DataDDS";
1454 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1465 sw.
start(
"HDF4RequestHandler::hdf4_build_dmr", dhi.
data[REQUEST_ID]);
1473 DDS dds(&factory, name_path(data_path),
"3.2");
1474 dds.filename(data_path);
1484 if(
true == _usecf) {
1486 if(
true == _pass_fileid)
1487 return hdf4_build_dmr_with_IDs(dhi);
1494 if (
true == _usecf) {
1500 sdfd = SDstart (data_path.c_str(), DFACC_READ);
1502 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1503 invalid_file_msg +=data_path;
1504 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1509 fileid = Hopen(data_path.c_str(), DFACC_READ,0);
1512 string invalid_file_msg=
"HDF4 Hopen error for the file ";
1513 invalid_file_msg +=data_path;
1514 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1518#ifdef USE_HDFEOS2_LIB
1523 HDFEOS2::File *eosfile =
nullptr;
1527 gridfd = GDopen(
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1531 string invalid_file_msg=
"HDF-EOS GDopen error for the file ";
1532 invalid_file_msg +=data_path;
1533 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
1538 swathfd = SWopen(
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1539 if (-1 == swathfd) {
1543 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
1544 invalid_file_msg +=data_path;
1545 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
1556 bool ecs_metadata =
true;
1558 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1559 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1561 if((
true == _disable_ecsmetadata_min)
1562 || (
true == _disable_ecsmetadata_all))
1563 ecs_metadata =
false;
1567 read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1571 Ancillary::read_ancillary_das(das, data_path);
1574 read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1578 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1581 if(eosfile !=
nullptr)
1588 read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1589 Ancillary::read_ancillary_das(das, data_path);
1592 read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1595 close_hdf4_fileid(sdfd,fileid,h4file);
1600 close_hdf4_fileid(sdfd,fileid,h4file);
1604 read_das(das, data_path);
1605 Ancillary::read_ancillary_das(das, data_path);
1606 read_dds(dds, data_path);
1610 Ancillary::read_ancillary_dds(dds, data_path);
1612 dds.transfer_attributes(&das);
1616 catch (InternalErr & e) {
1617 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1618 __FILE__, __LINE__);
1621 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1622 __FILE__, __LINE__);
1625 string s =
"unknown exception caught building HDF4 DataDDS";
1626 throw BESDapError(s,
true, unknown_error, __FILE__, __LINE__);
1639 DMR *dmr = bes_dmr.get_dmr();
1641 D4BaseTypeFactory MyD4TypeFactory;
1642 dmr->set_factory(&MyD4TypeFactory);
1646 dmr->build_using_dds(dds);
1659 dmr->set_factory(
nullptr);
1668 sw.
start(
"HDF4RequestHandler::hdf4_build_dmr_with_IDs", dhi.
data[REQUEST_ID]);
1676 DDS dds(&factory, name_path(data_path),
"3.2");
1677 dds.filename(data_path);
1688 sdfd = SDstart (data_path.c_str(), DFACC_READ);
1690 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1691 invalid_file_msg +=data_path;
1692 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1697 fileid = Hopen(data_path.c_str(), DFACC_READ,0);
1700 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1701 invalid_file_msg +=data_path;
1702 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1707#ifdef USE_HDFEOS2_LIB
1712 HDFEOS2::File *eosfile =
nullptr;
1715 gridfd = GDopen(
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1719 string invalid_file_msg=
"HDF4 SDstart error for the file ";
1720 invalid_file_msg +=data_path;
1721 invalid_file_msg +=
". It is very possible that this file is not an HDF4 file. ";
1727 swathfd = SWopen(
const_cast < char *
>(data_path.c_str()), DFACC_READ);
1728 if (-1 == swathfd) {
1732 string invalid_file_msg=
"HDF-EOS SWopen error for the file ";
1733 invalid_file_msg +=data_path;
1734 invalid_file_msg +=
". It is very possible that this file is not an HDF-EOS2 file. ";
1745 bool ecs_metadata =
true;
1747 if((
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataMin"))
1748 || (
true == HDFCFUtil::check_beskeys(
"H4.DisableECSMetaDataAll")))
1750 if((
true == _disable_ecsmetadata_min)
1751 || (
true == _disable_ecsmetadata_all))
1752 ecs_metadata =
false;
1756 read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1760 Ancillary::read_ancillary_das(das, data_path);
1763 read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1767 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1770 if(eosfile !=
nullptr)
1775 read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1776 Ancillary::read_ancillary_das(das, data_path);
1779 read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1782 close_hdf4_fileid(sdfd,fileid,h4file);
1786 if(h4file !=
nullptr)
1789 Ancillary::read_ancillary_dds(dds, data_path);
1791 dds.transfer_attributes(&das);
1797#ifdef USE_HDFEOS2_LIB
1818 DMR *dmr = bes_dmr.get_dmr();
1819 D4BaseTypeFactory MyD4TypeFactory;
1820 dmr->set_factory(&MyD4TypeFactory);
1821 dmr->build_using_dds(dds);
1822 auto hdf4_dmr =
new HDF4DMR(dmr);
1823#ifdef USE_HDFEOS2_LIB
1824 hdf4_dmr->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
1826 hdf4_dmr->setHDF4Dataset(sdfd,fileid);
1829 bes_dmr.set_dmr(hdf4_dmr);
1838 hdf4_dmr->set_factory(
nullptr);
1846 auto info =
dynamic_cast<BESInfo *
> (response);
1850 map < string, string > attrs;
1851 attrs[
"name"] = MODULE_NAME ;
1852 attrs[
"version"] = MODULE_VERSION ;
1853 list < string > services;
1855 if (services.empty() ==
false) {
1857 attrs[
"handles"] = handles;
1859 info->begin_tag(
"module", &attrs);
1860 info->end_tag(
"module");
1871 info->add_module(MODULE_NAME, MODULE_VERSION);
1876#ifdef USE_HDFEOS2_LIB
1877void close_fileid(
int sdfd,
int fileid,
int gridfd,
int swathfd,HDFSP:: File* h4file, HDFEOS2::File*eosfile) {
1878 if(h4file !=
nullptr)
1886 if(eosfile !=
nullptr)
1895void close_hdf4_fileid(
int sdfd,
int fileid,
HDFSP::File*h4file) {
1897 if(h4file !=
nullptr)
1909bool rw_das_cache_file(
const string & filename, DAS *das_ptr,
bool w_flag) {
1911 bool das_set_cache =
false;
1912 FILE *das_file =
nullptr;
1915 das_file = fopen(filename.c_str(),
"r");
1917 das_file = fopen(filename.c_str(),
"w");
1919 if(
nullptr == das_file) {
1920 if(ENOENT == errno) {
1923 if(
false == w_flag) {
1924 BESDEBUG(
"h4",
"DAS set cache key is true." << endl);
1925 das_set_cache =
true;
1929 throw BESInternalError(
"An error occurred trying to open a das cache file " + get_errno(), __FILE__, __LINE__);
1933 int fd_das = fileno(das_file);
1936 struct flock *l_das;
1938 l_das = lock(F_RDLCK);
1940 l_das = lock(F_WRLCK);
1943 if(fcntl(fd_das,F_SETLKW,l_das) == -1) {
1946 oss <<
"cache process: " << l_das->l_pid <<
" triggered a locking error: " << get_errno();
1950 if(
false == w_flag){
1952 BESDEBUG(
"h4",
"Obtaining DAS from the cache file" << endl);
1954 das_ptr->parse(das_file);
1957 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1959 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1962 throw InternalErr(__FILE__,__LINE__,
"Fail to parse the das from a das file.");
1967 BESDEBUG(
"h4",
"write DAS to a cache file" << endl);
1969 das_ptr->print(das_file);
1972 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1974 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1977 throw InternalErr(__FILE__,__LINE__,
"Fail to generate a das cache file.");
1983 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1985 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1991 return das_set_cache;
1996bool r_dds_cache_file(
const string & cache_filename, DDS *dds_ptr,
const string & hdf4_filename) {
1998 bool dds_set_cache =
false;
1999 FILE *dds_file =
nullptr;
2000 dds_file = fopen(cache_filename.c_str(),
"rb");
2002 if(
nullptr == dds_file) {
2003 if(ENOENT == errno) {
2007 dds_set_cache =
true;
2010 throw BESInternalError(
"An error occurred trying to open a dds cache file " + get_errno(), __FILE__, __LINE__);
2014 int fd_dds = fileno(dds_file);
2015 struct flock *l_dds;
2016 l_dds = lock(F_RDLCK);
2019 if(fcntl(fd_dds,F_SETLKW,l_dds) == -1) {
2022 oss <<
"cache process: " << l_dds->l_pid <<
" triggered a locking error: " << get_errno();
2027 HDFCFUtil::read_sp_sds_dds_cache(dds_file,dds_ptr,cache_filename,hdf4_filename);
2030 if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2032 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2036 throw InternalErr(__FILE__,__LINE__,
"Fail to generate a dds cache file.");
2039 if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2041 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2048 return dds_set_cache;
2052bool check_beskeys(
const string & key) {
2056 const string dosettrue =
"true";
2057 const string dosetyes =
"yes";
2060 if(
true == found ) {
2062 if( dosettrue == doset || dosetyes == doset )
2069bool get_beskeys(
const string & key,
string &key_value) {
2079int get_cachekey_int(
const string key) {
2086 if(
true == found ) {
2087 istringstream iss(doset);
2096cerr<<
"OK to pass pointer of a nullptr pointer "<<endl;
std::string get_symbolic_name() const
retrieve the symbolic name for this container
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
Holds a DDS object within the BES.
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Base exception class for the BES with basic string message.
informational response object
exception thrown if internal error encountered
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
static std::string lowercase(const std::string &s)
static std::string implode(const std::list< std::string > &values, char delim)
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
void close_fileid(hid_t fid)