bes Updated for version 3.20.13
HDF4RequestHandler.cc
1
2// -*- mode: c++; c-basic-offset:4 -*-
3
4// This file is part of hdf4_handler, a data handler for the OPeNDAP data
5// server.
6
7// Copyright (c) 2002,2003 OPeNDAP, Inc.
8// Author: James Gallagher <jgallagher@opendap.org>
9// Author: Muqun Yang <myang6@opendap.org>
10// This is free software; you can redistribute it and/or modify it under the
11// terms of the GNU Lesser General Public License as published by the Free
12// Software Foundation; either version 2.1 of the License, or (at your
13// option) any later version.
14//
15// This software is distributed in the hope that it will be useful, but
16// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
17// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
18// License for more details.
19//
20// You should have received a copy of the GNU Lesser General Public
21// License along with this library; if not, write to the Free Software
22// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23//
24// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
25
26// HDF4RequestHandler.cc
27
28#include <cstdlib>
29#include <sys/types.h>
30#include <sys/stat.h>
31#include <fcntl.h>
32#include <cerrno>
33#include <unistd.h>
34#include <iostream>
35#include <sstream>
36
37#include <libdap/DMR.h>
38#include <libdap/D4BaseTypeFactory.h>
39#include <BESDMRResponse.h>
40#include <libdap/mime_util.h>
41#include <libdap/InternalErr.h>
42#include <libdap/Ancillary.h>
43#include <libdap/debug.h>
44
45#include "HDF4RequestHandler.h"
46#include <BESResponseNames.h>
47#include <BESDapNames.h>
48#include <BESDASResponse.h>
49#include <BESDDSResponse.h>
50#include <BESDataDDSResponse.h>
51#include <BESInfo.h>
52#include <BESResponseHandler.h>
53#include <BESVersionInfo.h>
54#include <BESServiceRegistry.h>
55#include <BESUtil.h>
56#include <TheBESKeys.h>
57#include <libdap/InternalErr.h>
58#include <BESInternalError.h>
59#include <BESDapError.h>
60#include <BESStopWatch.h>
61#include <BESDebug.h>
62#include "BESDataNames.h"
63//#include <libdap/ConstraintEvaluator.h>
64#include <libdap/Ancillary.h>
65#include "config_hdf.h"
66
67#define HDF4_NAME "h4"
68#include "HE2CF.h"
69#include "HDF4_DDS.h"
70
71#include "HDF4_DMR.h"
72
73//#include "HDFCFUtil.h"
74#include "HDFFloat32.h"
75#include "HDFSPArray_RealField.h"
76
77#include "dodsutil.h"
78//#if 0
79#include <sys/time.h>
80//#endif
81
82using namespace std;
83using namespace libdap;
84
85bool check_beskeys(const string &);
86bool get_beskeys(const string &,string &);
87
88extern void read_das(DAS & das, const string & filename);
89extern void read_dds(DDS & dds, const string & filename);
90
91extern bool read_dds_hdfsp(DDS & dds, const string & filename,int32 sdfd, int32 fileid,HDFSP::File*h4file);
92
93extern bool read_das_hdfsp(DAS & das, const string & filename,int32 sdfd, int32 fileid,HDFSP::File**h4fileptr);
94
95extern void read_das_sds(DAS & das, const string & filename,int32 sdfd, bool ecs_metadata,HDFSP::File**h4fileptr);
96extern void read_dds_sds(DDS &dds, const string & filename,int32 sdfd, HDFSP::File*h4file,bool dds_set_cache);
97
98#ifdef USE_HDFEOS2_LIB
99
100void read_das_use_eos2lib(DAS & das, const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,bool ecs_metadata,HDFSP::File**h4file,HDFEOS2::File**eosfile);
101void read_dds_use_eos2lib(DDS & dds, const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,HDFSP::File*h4file,HDFEOS2::File*eosfile);
102void close_fileid(const int sdfd, const int fileid,const int gridfd, const int swathfd,HDFSP::File*h4file,HDFEOS2::File*eosfile);
103
104#endif
105
106void close_hdf4_fileid(const int sdfd,const int fileid,HDFSP::File*h4file);
107bool rw_das_cache_file(const string & filename, DAS *das_ptr,bool rw_flag);
108bool r_dds_cache_file(const string & cache_filename, DDS *dds_ptr,const string & hdf4_filename);
109
110// CF key
111bool HDF4RequestHandler::_usecf = false;
112
113// Keys to tune the performance -general
114bool HDF4RequestHandler::_pass_fileid = false;
115bool HDF4RequestHandler::_disable_structmeta = false;
116bool HDF4RequestHandler::_enable_special_eos = false;
117bool HDF4RequestHandler::_disable_scaleoffset_comp = false;
118bool HDF4RequestHandler::_disable_ecsmetadata_min = false;
119bool HDF4RequestHandler::_disable_ecsmetadata_all = false;
120
121
122// Keys to tune the performance - cache
123bool HDF4RequestHandler::_enable_eosgeo_cachefile = false;
124bool HDF4RequestHandler::_enable_data_cachefile = false;
125bool HDF4RequestHandler::_enable_metadata_cachefile= false;
126
127// Keys to handle vdata and vgroups
128bool HDF4RequestHandler::_enable_hybrid_vdata = false;
129bool HDF4RequestHandler::_enable_ceres_vdata = false;
130bool HDF4RequestHandler::_enable_vdata_attr = false;
131bool HDF4RequestHandler::_enable_vdata_desc_attr = false;
132bool HDF4RequestHandler::_disable_vdata_nameclashing_check = false;
133bool HDF4RequestHandler::_enable_vgroup_attr = false;
134
135// Misc. keys
136bool HDF4RequestHandler::_enable_check_modis_geo_file = false;
137bool HDF4RequestHandler::_enable_swath_grid_attr = false;
138bool HDF4RequestHandler::_enable_ceres_merra_short_name = false;
139bool HDF4RequestHandler::_enable_check_scale_offset_type = false;
140bool HDF4RequestHandler::_disable_swath_dim_map = false;
141
142// Cache path,prefix and size
143bool HDF4RequestHandler::_cache_latlon_path_exist =false;
144string HDF4RequestHandler::_cache_latlon_path ="";
145bool HDF4RequestHandler::_cache_latlon_prefix_exist =false;
146string HDF4RequestHandler::_cache_latlon_prefix ="";
147bool HDF4RequestHandler::_cache_latlon_size_exist =false;
148long HDF4RequestHandler::_cache_latlon_size =0;
149bool HDF4RequestHandler::_cache_metadata_path_exist =false;
150string HDF4RequestHandler::_cache_metadata_path ="";
151
152HDF4RequestHandler::HDF4RequestHandler(const string & name) :
153 BESRequestHandler(name) {
154 add_handler(DAS_RESPONSE, HDF4RequestHandler::hdf4_build_das);
155 add_handler(DDS_RESPONSE, HDF4RequestHandler::hdf4_build_dds);
156 add_handler(DATA_RESPONSE, HDF4RequestHandler::hdf4_build_data);
157 add_handler(DMR_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
158 add_handler(DAP4DATA_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
159 add_handler(HELP_RESPONSE, HDF4RequestHandler::hdf4_build_help);
160 add_handler(VERS_RESPONSE, HDF4RequestHandler::hdf4_build_version);
161
162 _usecf = check_beskeys("H4.EnableCF");
163
164 // The following keys are only effective when usecf is true.
165 // Keys to tune the performance -general
166 _pass_fileid = check_beskeys("H4.EnablePassFileID");
167 _disable_structmeta = check_beskeys("H4.DisableStructMetaAttr");
168 _enable_special_eos = check_beskeys("H4.EnableSpecialEOS");
169 _disable_scaleoffset_comp = check_beskeys("H4.DisableScaleOffsetComp");
170 _disable_ecsmetadata_min = check_beskeys("H4.DisableECSMetaDataMin");
171 _disable_ecsmetadata_all = check_beskeys("H4.DisableECSMetaDataAll");
172
173
174 // Keys to tune the performance - cache
175 _enable_eosgeo_cachefile = check_beskeys("H4.EnableEOSGeoCacheFile");
176 _enable_data_cachefile = check_beskeys("H4.EnableDataCacheFile");
177 _enable_metadata_cachefile = check_beskeys("H4.EnableMetaDataCacheFile");
178
179 // Keys to handle vdata and vgroups
180 _enable_hybrid_vdata = check_beskeys("H4.EnableHybridVdata");
181 _enable_ceres_vdata = check_beskeys("H4.EnableCERESVdata");
182 _enable_vdata_attr = check_beskeys("H4.EnableVdata_to_Attr");
183 _enable_vdata_desc_attr = check_beskeys("H4.EnableVdataDescAttr");
184 _disable_vdata_nameclashing_check = check_beskeys("H4.DisableVdataNameclashingCheck");
185 _enable_vgroup_attr = check_beskeys("H4.EnableVgroupAttr");
186
187 // Misc. keys
188 _enable_check_modis_geo_file = check_beskeys("H4.EnableCheckMODISGeoFile");
189 _enable_swath_grid_attr = check_beskeys("H4.EnableSwathGridAttr");
190 _enable_ceres_merra_short_name = check_beskeys("H4.EnableCERESMERRAShortName");
191 _enable_check_scale_offset_type = check_beskeys("H4.EnableCheckScaleOffsetType");
192
193 _disable_swath_dim_map = check_beskeys("H4.DisableSwathDimMap");
194
195 // Cache path etc.
196 _cache_latlon_path_exist =get_beskeys("HDF4.Cache.latlon.path",_cache_latlon_path);
197 _cache_latlon_prefix_exist =get_beskeys("HDF4.Cache.latlon.prefix",_cache_latlon_prefix);
198 string temp_cache_latlon_size;
199 _cache_latlon_size_exist =get_beskeys("HDF4.Cache.latlon.size",temp_cache_latlon_size);
200 if(_cache_latlon_size_exist == true) {
201 istringstream iss(temp_cache_latlon_size);
202 iss >> _cache_latlon_size;
203 }
204
205 _cache_metadata_path_exist =get_beskeys("H4.Cache.metadata.path",_cache_metadata_path);
206
207}
208
209#if 0
210HDF4RequestHandler::~HDF4RequestHandler() {
211}
212#endif
213
214bool HDF4RequestHandler::hdf4_build_das(BESDataHandlerInterface & dhi) {
215
216
217 BESStopWatch sw;
218 if (BESDebug::IsSet(TIMING_LOG_KEY))
219 sw.start("HDF4RequestHandler::hdf4_build_das", dhi.data[REQUEST_ID]);
220
221 if(true == _usecf) {
222
223 // Build DAP response only based on the HDF4 SD interfaces. Doing this
224 // way will save the use of other library open calls. Other library open
225 // calls may be expensive
226 // for an HDF4 file that only has variables created by SD interfaces.
227 // This optimization may be very useful for the aggreagation case that
228 // has many variables.
229 // Currently we only handle AIRS version 6 products. AIRS products
230 // are identified by their file names.
231 // We only obtain the filename. The path is stripped off.
232
233 string base_file_name = basename(dhi.container->access());
234
235 //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
236 // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
237 if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
238 && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
239 && (base_file_name.find(".v6.")!=string::npos)) {
240 return hdf4_build_das_cf_sds(dhi);
241
242 }
243 }
244
245 BESResponseObject *response = dhi.response_handler->get_response_object();
246 auto bdas = dynamic_cast<BESDASResponse *> (response);
247 if (!bdas)
248 throw BESInternalError("cast error", __FILE__, __LINE__);
249
250 try {
251 bdas->set_container(dhi.container->get_symbolic_name());
252 DAS *das = bdas->get_das();
253
254 string base_file_name = basename(dhi.container->access());
255
256 string accessed = dhi.container->access();
257
258 if (true == _usecf) {
259
260 int32 sdfd = -1;
261 int32 fileid = -1;
262
263 HDFSP::File *h4file = nullptr;
264
265 // Obtain HDF4 file IDs
266 //SDstart
267 sdfd = SDstart (accessed.c_str(), DFACC_READ);
268 if( -1 == sdfd){
269 string invalid_file_msg="HDF4 SDstart error for the file ";
270 invalid_file_msg +=accessed;
271 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
272 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
273 }
274
275 // H open
276 fileid = Hopen(accessed.c_str(), DFACC_READ,0);
277 if (-1 == fileid) {
278 SDend(sdfd);
279 string invalid_file_msg="HDF4 Hopen error for the file ";
280 invalid_file_msg +=accessed;
281 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
282 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
283 }
284
285#ifdef USE_HDFEOS2_LIB
286
287 int32 gridfd = -1;
288 int32 swathfd = -1;
289 HDFEOS2::File *eosfile = nullptr;
290 // Obtain HDF-EOS2 file IDs with the file open APIs.
291
292 // Grid open
293 gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
294 if (-1 == gridfd) {
295 SDend(sdfd);
296 Hclose(fileid);
297 string invalid_file_msg="HDF-EOS GDopen error for the file ";
298 invalid_file_msg +=accessed;
299 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
300 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
301 }
302
303 // Swath open
304 swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
305 if (-1 == swathfd) {
306 SDend(sdfd);
307 Hclose(fileid);
308 GDclose(gridfd);
309 string invalid_file_msg="HDF-EOS SWopen error for the file ";
310 invalid_file_msg +=accessed;
311 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
312 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
313 }
314
315 try {
316 bool ecs_metadata = !_disable_ecsmetadata_all;
317#if 0
318if(ecs_metadata == true)
319cerr<<"output ecs metadata "<<endl;
320else
321cerr<<"Don't output ecs metadata "<<endl;
322#endif
323 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
324 }
325 catch(...) {
326 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
327 throw;
328#if 0
329 //throw InternalErr(__FILE__,__LINE__,"read_das_use_eos2lib error");
330#endif
331 }
332 if(eosfile != nullptr)
333 delete eosfile;
334 GDclose(gridfd);
335 SWclose(swathfd);
336
337#else
338 try {
339 read_das_hdfsp(*das,accessed,sdfd,fileid,&h4file);
340 }
341 catch(...) {
342 close_hdf4_fileid(sdfd,fileid,h4file);
343 throw;
344 //throw InternalErr(__FILE__,__LINE__,"read_das_hdfsp error");
345 }
346#endif
347 close_hdf4_fileid(sdfd,fileid,h4file);
348 }
349 else
350 read_das(*das,accessed);
351
352 Ancillary::read_ancillary_das(*das, accessed);
353 bdas->clear_container();
354 }
355
356 catch (BESError & e) {
357 throw;
358 }
359 catch (InternalErr & e) {
360 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
361 __FILE__, __LINE__);
362 }
363 catch (Error & e) {
364 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
365 __FILE__, __LINE__);
366 }
367 catch (...) {
368 string s = "unknown exception caught building HDF4 DAS";
369 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
370 }
371
372 return true;
373}
374
375bool HDF4RequestHandler::hdf4_build_dds(BESDataHandlerInterface & dhi) {
376
377
378
379 BESStopWatch sw;
380 if (BESDebug::IsSet(TIMING_LOG_KEY))
381 sw.start("HDF4RequestHandler::hdf4_build_das", dhi.data[REQUEST_ID]);
382
383
384 if(true == _usecf) {
385 // Build DAP response only based on the HDF4 SD interfaces. Doing this
386 // way will save the use of other library open calls. Other library open
387 // calls may be expensive
388 // for an HDF4 file that only has variables created by SD interfaces.
389 // This optimization may be very useful for the aggreagation case that
390 // has many variables.
391 // Currently we only handle AIRS version 6 products. AIRS products
392 // are identified by their file names.
393 // We only obtain the filename. The path is stripped off.
394
395 string base_file_name = basename(dhi.container->access());
396
397 //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
398 // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
399 if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
400 && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
401 && (base_file_name.find(".v6.")!=string::npos)) {
402 return hdf4_build_dds_cf_sds(dhi);
403
404 }
405 }
406
407// This is for the performance check. Leave it now for future use. KY 2014-10-23
408#ifdef KENT2
409struct timeval start_time,end_time;
410gettimeofday(&start_time,nullptr);
411#endif
412
413 BESResponseObject *response = dhi.response_handler->get_response_object();
414 auto bdds = dynamic_cast<BESDDSResponse *> (response);
415 if (!bdds)
416 throw BESInternalError("cast error", __FILE__, __LINE__);
417
418 try {
419 bdds->set_container(dhi.container->get_symbolic_name());
420 DDS *dds = bdds->get_dds();
421#if 0
422 //ConstraintEvaluator & ce = bdds->get_ce();
423#endif
424
425 string accessed = dhi.container->access();
426 dds->filename(accessed);
427
428 auto das = new DAS;
429 BESDASResponse bdas(das);
430 bdas.set_container(dhi.container->get_symbolic_name());
431
432 if (true == _usecf) {
433
434 int32 sdfd = -1;
435 int32 fileid = -1;
436 HDFSP::File *h4file = nullptr;
437
438 // Obtain HDF4 file IDs
439 //SDstart
440 sdfd = SDstart (accessed.c_str(), DFACC_READ);
441 if( -1 == sdfd){
442 string invalid_file_msg="HDF4 SDstart error for the file ";
443 invalid_file_msg +=accessed;
444 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
445 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
446 }
447
448 // H open
449 fileid = Hopen(accessed.c_str(), DFACC_READ,0);
450 if (-1 == fileid) {
451 SDend(sdfd);
452 string invalid_file_msg="HDF4 Hopen error for the file ";
453 invalid_file_msg +=accessed;
454 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
455 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
456 }
457
458#ifdef USE_HDFEOS2_LIB
459
460 int32 gridfd = -1;
461 int32 swathfd = -1;
462
463 HDFEOS2::File *eosfile = nullptr;
464
465 // Obtain HDF-EOS2 file IDs with the file open APIs.
466 // Grid open
467 gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
468 if (-1 == gridfd) {
469 SDend(sdfd);
470 Hclose(fileid);
471 string invalid_file_msg="HDF-EOS GDopen error for the file ";
472 invalid_file_msg +=accessed;
473 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
474 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
475 }
476
477 // Swath open
478 swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
479 if (-1 == swathfd) {
480 SDend(sdfd);
481 Hclose(fileid);
482 GDclose(gridfd);
483 string invalid_file_msg="HDF-EOS SWopen error for the file ";
484 invalid_file_msg +=accessed;
485 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
486 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
487 }
488
489 try {
490 bool ecs_metadata = !_disable_ecsmetadata_all;
491 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
492 Ancillary::read_ancillary_das(*das, accessed);
493
494 // Pass file pointer(h4file, eosfile) from DAS to DDS.
495 read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
496 }
497 catch(...) {
498 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
499 throw;
500 }
501
502 if(eosfile != nullptr)
503 delete eosfile;
504
505 GDclose(gridfd);
506 SWclose(swathfd);
507
508#else
509 try {
510 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
511 Ancillary::read_ancillary_das(*das, accessed);
512
513 // Pass file pointer(h4file) from DAS to DDS.
514 read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
515 }
516 catch(...) {
517 close_hdf4_fileid(sdfd,fileid,h4file);
518 throw;
519 }
520
521#endif
522 close_hdf4_fileid(sdfd,fileid,h4file);
523 }
524 else {
525 read_das(*das, accessed);
526 Ancillary::read_ancillary_das(*das, accessed);
527 read_dds(*dds, accessed);
528 }
529
530// Leave it for future performance tests. KY 2014-10-23
531#ifdef KENT2
532gettimeofday(&end_time,nullptr);
533int total_time_spent = (end_time.tv_sec - start_time.tv_sec)*1000000 +end_time.tv_usec-start_time.tv_usec;
534cerr<<"total time spent for DDS buld is "<<total_time_spent<< "micro seconds "<<endl;
535#endif
536
537 Ancillary::read_ancillary_dds(*dds, accessed);
538
539 dds->transfer_attributes(das);
540
541 bdds->set_constraint(dhi);
542
543 bdds->clear_container();
544 }
545 catch (BESError & e) {
546 throw;
547 }
548 catch (InternalErr & e) {
549 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
550 __FILE__, __LINE__);
551 }
552 catch (Error & e) {
553 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
554 __FILE__, __LINE__);
555 }
556 catch (...) {
557 string s = "unknown exception caught building HDF4 DDS";
558 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
559 }
560
561 return true;
562}
563
564bool HDF4RequestHandler::hdf4_build_data(BESDataHandlerInterface & dhi) {
565
566
567 BESStopWatch sw;
568 if (BESDebug::IsSet(TIMING_LOG_KEY))
569 sw.start("HDF4RequestHandler::hdf4_build_data", dhi.data[REQUEST_ID]);
570
571
572 int32 sdfd = -1;
573 int32 fileid = -1;
574
575 // Since passing file IDs requires to use the derived class and it
576 // causes the management of code structure messy, we first handle this with
577 // another method.
578 if(true == _usecf) {
579 // Build DAP response only based on the HDF4 SD interfaces. Doing this
580 // way will save the use of other library open calls. Other library open
581 // calls may be expensive
582 // for an HDF4 file that only has variables created by SD interfaces.
583 // This optimization may be very useful for the aggreagation case that
584 // has many variables.
585 // Currently we only handle AIRS version 6 products. AIRS products
586 // are identified by their file names.
587 // We only obtain the filename. The path is stripped off.
588
589 string base_file_name = basename(dhi.container->access());
590 //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
591 // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
592 if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
593 && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
594 && (base_file_name.find(".v6.")!=string::npos)) {
595
596 BESDEBUG("h4", "Coming to read the data of AIRS level 3 or level 2 products." << endl);
597
598 if(true == _pass_fileid)
599 return hdf4_build_data_cf_sds_with_IDs(dhi);
600 else
601 return hdf4_build_data_cf_sds(dhi);
602
603 }
604
605 if(true == _pass_fileid)
606 return hdf4_build_data_with_IDs(dhi);
607
608 }
609
610 BESResponseObject *response = dhi.response_handler->get_response_object();
611 auto bdds = dynamic_cast<BESDataDDSResponse *> (response);
612
613 if (!bdds)
614 throw BESInternalError("cast error", __FILE__, __LINE__);
615
616 try {
617 bdds->set_container(dhi.container->get_symbolic_name());
618
619 DDS *dds = bdds->get_dds();
620
621 // Not sure why keep the following line, it is not used.
622 //ConstraintEvaluator & ce = bdds->get_ce();
623
624 string accessed = dhi.container->access();
625 dds->filename(accessed);
626
627 auto das = new DAS;
628 BESDASResponse bdas(das);
629 bdas.set_container(dhi.container->get_symbolic_name());
630
631 if (true == _usecf) {
632
633 HDFSP::File *h4file = nullptr;
634
635 // Obtain HDF4 file IDs
636 //SDstart
637 sdfd = SDstart (accessed.c_str(), DFACC_READ);
638 if( -1 == sdfd) {
639 string invalid_file_msg="HDF4 SDstart error for the file ";
640 invalid_file_msg +=accessed;
641 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
642 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
643 }
644
645 // H open
646 fileid = Hopen(accessed.c_str(), DFACC_READ,0);
647 if (-1 == fileid) {
648 SDend(sdfd);
649 string invalid_file_msg="HDF4 Hopen error for the file ";
650 invalid_file_msg +=accessed;
651 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
652 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
653 }
654
655
656#ifdef USE_HDFEOS2_LIB
657
658 int32 gridfd = -1;
659 int32 swathfd = -1;
660 HDFEOS2::File *eosfile = nullptr;
661 // Obtain HDF-EOS2 file IDs with the file open APIs.
662
663 // Grid open
664 gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
665 if (-1 == gridfd) {
666 SDend(sdfd);
667 Hclose(fileid);
668 string invalid_file_msg="HDF-EOS GDopen error for the file ";
669 invalid_file_msg +=accessed;
670 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
671 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
672 }
673
674 // Swath open
675 swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
676 if (-1 == swathfd) {
677 SDend(sdfd);
678 Hclose(fileid);
679 GDclose(gridfd);
680 string invalid_file_msg="HDF-EOS SWopen error for the file ";
681 invalid_file_msg +=accessed;
682 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
683 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
684
685 }
686
687 try {
688
689 // Here we will check if ECS_Metadata key if set. For DataDDS,
690 // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
691 // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
692 bool ecs_metadata = true;
693#if 0
694 if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
695 || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
696 ecs_metadata = false;
697#endif
698 if((true == _disable_ecsmetadata_min)
699 || (true == _disable_ecsmetadata_all))
700 ecs_metadata = false;
701
702 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
703 Ancillary::read_ancillary_das(*das, accessed);
704
705 // Pass file pointer(h4file, eosfile) from DAS to DDS.
706 read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
707 }
708 catch(...) {
709 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
710 throw;
711 }
712
713 if(eosfile != nullptr)
714 delete eosfile;
715 GDclose(gridfd);
716 SWclose(swathfd);
717
718#else
719 try {
720 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
721 Ancillary::read_ancillary_das(*das, accessed);
722
723 // Pass file pointer(h4file) from DAS to DDS.
724 read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
725 }
726 catch(...) {
727 close_hdf4_fileid(sdfd,fileid,h4file);
728 throw;
729 }
730#endif
731 close_hdf4_fileid(sdfd,fileid,h4file);
732 }
733 else {
734 read_das(*das, accessed);
735 Ancillary::read_ancillary_das(*das, accessed);
736 read_dds(*dds, accessed);
737 }
738
739 Ancillary::read_ancillary_dds(*dds, accessed);
740 dds->transfer_attributes(das);
741 bdds->set_constraint(dhi);
742 bdds->clear_container();
743
744 }
745
746 catch (BESError & e) {
747 throw;
748 }
749 catch (InternalErr & e) {
750 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
751 __FILE__, __LINE__);
752 }
753 catch (Error & e) {
754 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
755 __FILE__, __LINE__);
756 }
757 catch (...) {
758 string s = "unknown exception caught building DAP2 Data Response from an HDF4 data resource";
759 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
760 }
761
762 return true;
763}
764
765bool HDF4RequestHandler::hdf4_build_data_with_IDs(BESDataHandlerInterface & dhi) {
766
767 BESStopWatch sw;
768 if (BESDebug::IsSet(TIMING_LOG_KEY))
769 sw.start("HDF4RequestHandler::hdf4_build_data_with_IDs", dhi.data[REQUEST_ID]);
770
771 int32 sdfd = -1;
772 int32 fileid = -1;
773 HDFSP::File *h4file = nullptr;
774#ifdef USE_HDFEOS2_LIB
775 int32 gridfd = -1;
776 int32 swathfd = -1;
777 HDFEOS2::File *eosfile = nullptr;
778#endif
779
780 BESResponseObject *response = dhi.response_handler->get_response_object();
781 auto bdds = dynamic_cast<BESDataDDSResponse *> (response);
782
783 if (!bdds)
784 throw BESInternalError("cast error", __FILE__, __LINE__);
785
786 try {
787 bdds->set_container(dhi.container->get_symbolic_name());
788
789 // Create a new HDF4DDS object.
790 HDF4DDS *hdds = new HDF4DDS(bdds->get_dds());
791
792 // delete the old object.
793 delete bdds->get_dds();
794
795 bdds->set_dds(hdds);
796
797#if 0
798 //ConstraintEvaluator & ce = bdds->get_ce();
799#endif
800
801 string accessed = dhi.container->access();
802 hdds->filename(accessed);
803
804 auto das = new DAS;
805 BESDASResponse bdas(das);
806 bdas.set_container(dhi.container->get_symbolic_name());
807
808
809 // Obtain HDF4 file IDs
810 //SDstart
811 sdfd = SDstart (accessed.c_str(), DFACC_READ);
812 if( -1 == sdfd) {
813 string invalid_file_msg="HDF4 SDstart error for the file ";
814 invalid_file_msg +=accessed;
815 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
816 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
817 }
818 // H open
819 fileid = Hopen(accessed.c_str(), DFACC_READ,0);
820 if (-1 == fileid) {
821 SDend(sdfd);
822 string invalid_file_msg="HDF4 Hopen error for the file ";
823 invalid_file_msg +=accessed;
824 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
825 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
826 }
827
828#ifdef USE_HDFEOS2_LIB
829
830
831 // Obtain HDF-EOS2 file IDs with the file open APIs.
832
833 // Grid open
834 gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
835 if (-1 == gridfd) {
836 SDend(sdfd);
837 Hclose(fileid);
838 string invalid_file_msg="HDF-EOS GDopen error for the file ";
839 invalid_file_msg +=accessed;
840 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
841 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
842 }
843
844 // Swath open
845 swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
846 if (-1 == swathfd) {
847 SDend(sdfd);
848 Hclose(fileid);
849 GDclose(gridfd);
850 string invalid_file_msg="HDF-EOS SWopen error for the file ";
851 invalid_file_msg +=accessed;
852 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
853 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
854 }
855
856 hdds->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
857
858 // Here we will check if ECS_Metadata key if set. For DataDDS,
859 // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
860 // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
861 bool ecs_metadata = true;
862#if 0
863 if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
864 || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
865#endif
866 if((true == _disable_ecsmetadata_min)
867 || (true == _disable_ecsmetadata_all))
868 ecs_metadata = false;
869
870 read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
871
872#if 0
873 //read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,true);
874#endif
875
876 Ancillary::read_ancillary_das(*das, accessed);
877
878 // Pass file pointer(h4file, eosfile) from DAS to DDS.
879 read_dds_use_eos2lib(*hdds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
880
881 if(eosfile != nullptr)
882 delete eosfile;
883
884#else
885 hdds->setHDF4Dataset(sdfd,fileid);
886 read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
887 Ancillary::read_ancillary_das(*das, accessed);
888
889 // Pass file pointer(h4file) from DAS to DDS.
890 read_dds_hdfsp(*hdds, accessed,sdfd,fileid,h4file);
891#endif
892 if(h4file != nullptr)
893 delete h4file;
894
895 Ancillary::read_ancillary_dds(*hdds, accessed);
896
897 hdds->transfer_attributes(das);
898
899 bdds->set_constraint(dhi);
900
901 bdds->clear_container();
902
903
904// File IDs are closed by the derived class.
905#if 0
906 if(true == usecf) {
907#ifdef USE_HDFEOS2_LIB
908 GDclose(gridfd);
909 SWclose(swathfd);
910
911#endif
912 SDend(sdfd);
913 Hclose(fileid);
914 }
915#endif
916 }
917
918 catch (InternalErr & e) {
919 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
920 __FILE__, __LINE__);
921 }
922 catch (Error & e) {
923 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
924 __FILE__, __LINE__);
925 }
926 catch (...) {
927#ifdef USE_HDFEOS2_LIB
928 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
929#else
930 close_hdf4_fileid(sdfd,fileid,h4file);
931#endif
932 string s = "unknown exception caught building DAP2 data response from an HDF4 data resource";
933 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
934 }
935
936 return true;
937}
938
939// Special function to build DDS for HDF4 SDS-only DDS. One can turn
940// on cache for this function. Currently only AIRS version 6 is supported.
941bool HDF4RequestHandler::hdf4_build_dds_cf_sds(BESDataHandlerInterface &dhi){
942
943 int32 sdfd = -1;
944 HDFSP::File *h4file = nullptr;
945
946 BESResponseObject *response = dhi.response_handler->get_response_object();
947 auto bdds = dynamic_cast<BESDDSResponse *> (response);
948
949 if (!bdds)
950 throw BESInternalError("cast error", __FILE__, __LINE__);
951
952 try {
953 bdds->set_container(dhi.container->get_symbolic_name());
954
955 DDS *dds = bdds->get_dds();
956
957#if 0
958 //BaseTypeFactory* factory = new BaseTypeFactory ;
959 //dds->set_factory( factory ) ;
960#endif
961
962 string accessed = dhi.container->access();
963 dds->filename(accessed);
964
965 auto das = new DAS;
966 BESDASResponse bdas(das);
967
968 // Check and set up dds and das cache files.
969 string base_file_name = basename(dhi.container->access());
970 bool das_set_cache = false;
971 bool dds_set_cache = false;
972 bool dds_das_get_cache = false;
973 string das_filename;
974 string dds_filename;
975
976 // Check if the EnableMetaData key is set.
977#if 0
978 string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
979 bool turn_on_enable_mcache_key= false;
980 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
981 if(true == turn_on_enable_mcache_key) {// the EnableMetaData key is set
982#endif
983 if(true == _enable_metadata_cachefile) {// the EnableMetaData key is set
984
985
986#if 0
987 string md_cache_dir;
988 bool found = false;
989 string key = "H4.Cache.metadata.path";
990 TheBESKeys::TheKeys()->get_value(key,md_cache_dir,found);
991#endif
992 if(true == _cache_metadata_path_exist) {// the metadata.path key is set
993
994 // Create DAS and DDS cache file names
995 das_filename = _cache_metadata_path + "/" + base_file_name +"_das";
996 dds_filename = _cache_metadata_path + "/" + base_file_name +"_dds";
997
998 // When the returned value das_set_cache is false, read DAS from the cache file.
999 // Otherwise, do nothing.
1000 das_set_cache = rw_das_cache_file(das_filename,das,false);
1001
1002 // When the returned value dds_set_cache is false, read DDS from the cache file.
1003 // Otherwise, do nothing.
1004 dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1005
1006 // Set the flag to obtain DDS and DAS from the cache files.
1007 // Here, we require that both DDS and DAS should be cached.
1008 // We don't support caching one and not caching another.
1009 if((false == das_set_cache)&&(false == dds_set_cache))
1010 dds_das_get_cache = true;
1011 }
1012 }
1013
1014
1015 // We need to go back to retrieve DDS, DAS from the HDF file.
1016 if(false == dds_das_get_cache) {
1017
1018 // Obtain SD ID, this is the only ID we need to use.
1019 sdfd = SDstart (accessed.c_str(), DFACC_READ);
1020 if( -1 == sdfd) {
1021
1022 string invalid_file_msg="HDF4 SDstart error for the file ";
1023 invalid_file_msg +=accessed;
1024 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1025 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1026 }
1027
1028
1029 // Here we will check if ECS_Metadata key if set. For DDS and DAS,
1030 // only when the DisableECSMetaDataAll key is set, ecs_metadata is off.
1031 bool ecs_metadata = !_disable_ecsmetadata_all;
1032#if 0
1033 bool ecs_metadata = true;
1034 if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1035 || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1036 ecs_metadata = false;
1037#endif
1038
1039 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1040
1041 Ancillary::read_ancillary_das(*das, accessed);
1042
1043 // Pass file pointer(h4file) from DAS to DDS, also pass the flag
1044 // dds_set_cache to indicate if the handler needs to cache DDS.
1045 read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1046
1047 // We also need to cache DAS if das_set_cache is set.
1048 if(true == das_set_cache) {
1049 if(das_filename =="")
1050 throw InternalErr(__FILE__,__LINE__,"DAS cache file name should be set ");
1051 rw_das_cache_file(das_filename,das,true);
1052 }
1053
1054 }
1055
1056 Ancillary::read_ancillary_dds(*dds, accessed);
1057 dds->transfer_attributes(das);
1058 bdds->set_constraint(dhi);
1059
1060 bdds->clear_container();
1061
1062 if(h4file != nullptr)
1063 delete h4file;
1064
1065 if(sdfd != -1)
1066 SDend(sdfd);
1067#if 0
1068 // Cannot(should not) delete factory here. factory will be released by DAP? or errors occur.
1069 //if(factory != nullptr)
1070 // delete factory;
1071#endif
1072
1073 }
1074
1075 catch (BESError & e) {
1076 throw;
1077 }
1078 catch (InternalErr & e) {
1079 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1080 __FILE__, __LINE__);
1081 }
1082 catch (Error & e) {
1083 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1084 __FILE__, __LINE__);
1085 }
1086 catch (...) {
1087 if(sdfd != -1)
1088 SDend(sdfd);
1089 if(h4file != nullptr)
1090 delete h4file;
1091 string s = "unknown exception caught building HDF4 DataDDS";
1092 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1093 }
1094
1095 return true;
1096}
1097
1098// Special function to build DAS for HDF4 SDS-only DDS. One can turn
1099// on cache for this function. Currently only AIRS version 6 is supported.
1100bool HDF4RequestHandler::hdf4_build_das_cf_sds(BESDataHandlerInterface &dhi){
1101
1102 int32 sdfd = -1;
1103 HDFSP::File *h4file = nullptr;
1104
1105 BESResponseObject *response = dhi.response_handler->get_response_object();
1106 auto bdas = dynamic_cast<BESDASResponse *> (response);
1107
1108 if (!bdas)
1109 throw BESInternalError("cast error", __FILE__, __LINE__);
1110
1111 try {
1112 bdas->set_container(dhi.container->get_symbolic_name());
1113
1114 auto das = bdas->get_das();
1115 string base_file_name = basename(dhi.container->access());
1116
1117 string accessed = dhi.container->access();
1118
1119 // Check if the enable metadata cache key is set and set the key appropriate.
1120 bool das_set_cache = false;
1121 bool das_get_cache = false;
1122 string das_filename;
1123#if 0
1124 string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
1125 bool turn_on_enable_mcache_key= false;
1126 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1127 if(true == turn_on_enable_mcache_key) { // find the metadata cache key.
1128#endif
1129 if(true == _enable_metadata_cachefile) { // find the metadata cache key.
1130#if 0
1131 string md_cache_dir;
1132 string key = "H4.Cache.metadata.path";
1133 bool found = false;
1134 TheBESKeys::TheKeys()->get_value(key,md_cache_dir,found);
1135 if(true == found) { // Also find the metadata cache.
1136#endif
1137 if(true == _cache_metadata_path_exist) {
1138
1139 // Create the DAS cache file name.
1140 das_filename = _cache_metadata_path + "/" + base_file_name +"_das";
1141
1142 // Read the DAS from the cached file, if das_set_cache is false.
1143 // When the das_set_cache is true, need to create a das cache file.
1144 das_set_cache = rw_das_cache_file(das_filename,das,false);
1145
1146 // Not set cache, must get the das from cache, so das_get_cache should be true.
1147 if(false == das_set_cache)
1148 das_get_cache = true;
1149 }
1150 }
1151
1152 // Need to retrieve DAS from the HDF4 file.
1153 if(false == das_get_cache) {
1154 // Obtain SD ID.
1155 sdfd = SDstart (accessed.c_str(), DFACC_READ);
1156 if( -1 == sdfd){
1157 string invalid_file_msg="HDF4 SDstart error for the file ";
1158 invalid_file_msg +=accessed;
1159 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1160 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1161 }
1162
1163
1164#if 0
1165 bool ecs_metadata = true;
1166 if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1167 || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1168 ecs_metadata = false;
1169#endif
1170 // Here we will check if ECS_Metadata key if set. For DDS and DAS,
1171 // only when the DisableECSMetaDataAll key is set, ecs_metadata is off.
1172 bool ecs_metadata = !_disable_ecsmetadata_all;
1173
1174 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1175 Ancillary::read_ancillary_das(*das, accessed);
1176
1177 // Generate DAS cache file if the cache flag is set.
1178 if(true == das_set_cache)
1179 rw_das_cache_file(das_filename,das,true);
1180
1181 }
1182
1183 bdas->clear_container();
1184
1185 if(h4file != nullptr)
1186 delete h4file;
1187
1188 if(sdfd != -1)
1189 SDend(sdfd);
1190
1191 }
1192
1193 catch (BESError & e) {
1194 throw;
1195 }
1196 catch (InternalErr & e) {
1197 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1198 __FILE__, __LINE__);
1199 }
1200 catch (Error & e) {
1201 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1202 __FILE__, __LINE__);
1203 }
1204 catch (...) {
1205 if(sdfd != -1)
1206 SDend(sdfd);
1207 if(h4file != nullptr)
1208 delete h4file;
1209 string s = "unknown exception caught building HDF4 DataDDS";
1210 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1211 }
1212 return true;
1213}
1214// Special function to build data for HDF4 SDS-only DDS. One can turn
1215// on cache for this function. Currently only AIRS version 6 is supported.
1216bool HDF4RequestHandler::hdf4_build_data_cf_sds(BESDataHandlerInterface &dhi){
1217
1218 int32 sdfd = -1;
1219 HDFSP::File *h4file = nullptr;
1220
1221 BESResponseObject *response = dhi.response_handler->get_response_object();
1222 auto bdds = dynamic_cast<BESDataDDSResponse *> (response);
1223
1224 if (!bdds)
1225 throw BESInternalError("cast error", __FILE__, __LINE__);
1226
1227 try {
1228 bdds->set_container(dhi.container->get_symbolic_name());
1229
1230 DDS *dds = bdds->get_dds();
1231
1232 string accessed = dhi.container->access();
1233 dds->filename(accessed);
1234
1235 auto das = new DAS;
1236 BESDASResponse bdas(das);
1237 bdas.set_container(dhi.container->get_symbolic_name());
1238
1239#if 0
1240 //BaseTypeFactory* factory = new BaseTypeFactory ;
1241 //dds->set_factory( factory ) ;
1242#endif
1243
1244 string base_file_name = basename(dhi.container->access());
1245
1246 bool das_set_cache = false;
1247 bool dds_set_cache = false;
1248 bool dds_das_get_cache = false;
1249 string das_filename;
1250 string dds_filename;
1251
1252#if 0
1253 string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
1254 bool _disable_ecsmetadata_allturn_on_enable_mcache_key= false;
1255 turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1256 if(true == turn_on_enable_mcache_key) {
1257#endif
1258 if(true == _enable_metadata_cachefile) {
1259
1260 if(true == _cache_metadata_path_exist) {
1261 BESDEBUG("h4", "H4.Cache.metadata.path key is set and metadata cache key is set." << endl);
1262
1263 // Notice, since the DAS output may be different between DAS/DDS service and DataDDS service.
1264 // See comments about ecs_metadata below.
1265 // So we create a different DAS file. This can be optimized in the future if necessary.
1266 das_filename = _cache_metadata_path + "/" + base_file_name +"_das_dd";
1267 dds_filename = _cache_metadata_path + "/" + base_file_name +"_dds";
1268
1269 // If das_set_cache is true, data is read from the DAS cache.
1270 das_set_cache = rw_das_cache_file(das_filename,das,false);
1271
1272 // If dds_set_cache is true, data is read from the DDS cache.
1273 dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1274 // Need to set a flag to generate DAS and DDS cache files
1275 if((false == das_set_cache)&&(false == dds_set_cache))
1276 dds_das_get_cache = true;
1277 }
1278 }
1279 if(false == dds_das_get_cache) {
1280
1281 // Obtain HDF4 SD ID
1282 sdfd = SDstart (accessed.c_str(), DFACC_READ);
1283 if( -1 == sdfd){
1284 string invalid_file_msg="HDF4 SDstart error for the file ";
1285 invalid_file_msg +=accessed;
1286 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1287 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1288 }
1289
1290
1291 // Here we will check if ECS_Metadata key if set. For DataDDS,
1292 // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1293 // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1294 bool ecs_metadata = true;
1295#if 0
1296 if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1297 || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1298#endif
1299 if((true == _disable_ecsmetadata_min)
1300 || (true == _disable_ecsmetadata_all))
1301 ecs_metadata = false;
1302
1303 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1304
1305 Ancillary::read_ancillary_das(*das, accessed);
1306
1307 // Need to write DAS to a cache file.
1308 if(true == das_set_cache) {
1309 rw_das_cache_file(das_filename,das,true);
1310 }
1311
1312 // Pass file pointer h4file from DAS to DDS. Also need to pass the
1313 // flag that indicates if a DDS file needs to be created.
1314 read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1315
1316 }
1317
1318 Ancillary::read_ancillary_dds(*dds, accessed);
1319
1320 dds->transfer_attributes(das);
1321
1322 bdds->set_constraint(dhi);
1323
1324 bdds->clear_container();
1325
1326 if(h4file != nullptr)
1327 delete h4file;
1328
1329 if(sdfd != -1)
1330 SDend(sdfd);
1331 }
1332
1333 catch (BESError & e) {
1334 throw;
1335 }
1336 catch (InternalErr & e) {
1337 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1338 __FILE__, __LINE__);
1339 }
1340 catch (Error & e) {
1341 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1342 __FILE__, __LINE__);
1343 }
1344 catch (...) {
1345 if(sdfd != -1)
1346 SDend(sdfd);
1347 if(h4file != nullptr)
1348 delete h4file;
1349 string s = "unknown exception caught building HDF4 DataDDS";
1350 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1351 }
1352
1353
1354 return true;
1355}
1356
1357// Notice the metadata cache doesn't apply when the EnableFileID key is set.
1358// This is because the file ID generated by creating DDS/DAS WO cache is needed to access the data.
1359// So to make this work at any time, we have to create SDS ID even when the EnableFileID key is set.
1360// This is against the purpose of EnableFileID key. To acheieve the same purpose for AIRS,
1361// one can set the DataCache key, the performance is similar or even better than just using the EnableFileID key.
1362bool HDF4RequestHandler::hdf4_build_data_cf_sds_with_IDs(BESDataHandlerInterface &dhi){
1363
1364 int32 sdfd = -1;
1365 HDFSP::File *h4file = nullptr;
1366
1367 BESResponseObject *response = dhi.response_handler->get_response_object();
1368 auto bdds = dynamic_cast<BESDataDDSResponse *> (response);
1369
1370 if (!bdds)
1371 throw BESInternalError("cast error", __FILE__, __LINE__);
1372
1373 try {
1374 bdds->set_container(dhi.container->get_symbolic_name());
1375
1376 auto hdds = new HDF4DDS(bdds->get_dds());
1377
1378
1379 delete bdds->get_dds();
1380
1381 bdds->set_dds(hdds);
1382
1383
1384 string accessed = dhi.container->access();
1385 hdds->filename(accessed);
1386
1387 auto das = new DAS;
1388 BESDASResponse bdas(das);
1389 bdas.set_container(dhi.container->get_symbolic_name());
1390
1391 //Obtain SD ID.
1392 sdfd = SDstart (accessed.c_str(), DFACC_READ);
1393 if( -1 == sdfd) {
1394 string invalid_file_msg="HDF4 SDstart error for the file ";
1395 invalid_file_msg +=accessed;
1396 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1397 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1398 }
1399
1400 hdds->setHDF4Dataset(sdfd,-1);
1401
1402 // Here we will check if ECS_Metadata key if set. For DataDDS,
1403 // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1404 // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1405 bool ecs_metadata = true;
1406#if 0
1407 if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1408 || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1409#endif
1410 if((true == _disable_ecsmetadata_min)
1411 || (true == _disable_ecsmetadata_all))
1412 ecs_metadata = false;
1413
1414 read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1415#if 0
1416 //read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,true);
1417#endif
1418
1419 Ancillary::read_ancillary_das(*das, accessed);
1420
1421 // Pass file pointer(h4file, eosfile) from DAS to DDS.
1422 read_dds_sds(*hdds, accessed,sdfd,h4file,false);
1423
1424 if(h4file != nullptr)
1425 delete h4file;
1426
1427 Ancillary::read_ancillary_dds(*hdds, accessed);
1428
1429 hdds->transfer_attributes(das);
1430
1431 bdds->set_constraint(dhi);
1432
1433 bdds->clear_container();
1434
1435 }
1436
1437 catch (BESError & e) {
1438 throw;
1439 }
1440 catch (InternalErr & e) {
1441 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1442 __FILE__, __LINE__);
1443 }
1444 catch (Error & e) {
1445 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1446 __FILE__, __LINE__);
1447 }
1448 catch (...) {
1449 if(sdfd != -1)
1450 SDend(sdfd);
1451 if(h4file != nullptr)
1452 delete h4file;
1453 string s = "unknown exception caught building HDF4 DataDDS";
1454 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1455 }
1456
1457
1458 return true;
1459}
1460bool HDF4RequestHandler::hdf4_build_dmr(BESDataHandlerInterface &dhi)
1461{
1462
1463 BESStopWatch sw;
1464 if (BESDebug::IsSet(TIMING_LOG_KEY))
1465 sw.start("HDF4RequestHandler::hdf4_build_dmr", dhi.data[REQUEST_ID]);
1466
1467 // Because this code does not yet know how to build a DMR directly, use
1468 // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
1469 // First step, build the 'full DDS'
1470 string data_path = dhi.container->access();
1471
1472 BaseTypeFactory factory;
1473 DDS dds(&factory, name_path(data_path), "3.2");
1474 dds.filename(data_path);
1475
1476 DAS das;
1477
1478 int32 sdfd = -1;
1479 int32 fileid = -1;
1480
1481 // Since passing file IDs requires to use the derived class and it
1482 // causes the management of code structure messy, we first handle this with
1483 // another method.
1484 if(true == _usecf) {
1485
1486 if(true == _pass_fileid)
1487 return hdf4_build_dmr_with_IDs(dhi);
1488
1489 }
1490
1491
1492 try {
1493
1494 if (true == _usecf) {
1495
1496 HDFSP::File *h4file = nullptr;
1497
1498 // Obtain HDF4 file IDs
1499 //SDstart
1500 sdfd = SDstart (data_path.c_str(), DFACC_READ);
1501 if( -1 == sdfd){
1502 string invalid_file_msg="HDF4 SDstart error for the file ";
1503 invalid_file_msg +=data_path;
1504 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1505 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1506 }
1507
1508 // H open
1509 fileid = Hopen(data_path.c_str(), DFACC_READ,0);
1510 if (-1 == fileid) {
1511 SDend(sdfd);
1512 string invalid_file_msg="HDF4 Hopen error for the file ";
1513 invalid_file_msg +=data_path;
1514 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1515 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1516 }
1517
1518#ifdef USE_HDFEOS2_LIB
1519
1520 int32 gridfd = -1;
1521 int32 swathfd = -1;
1522
1523 HDFEOS2::File *eosfile = nullptr;
1524
1525 // Obtain HDF-EOS2 file IDs with the file open APIs.
1526 // Grid open
1527 gridfd = GDopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1528 if (-1 == gridfd) {
1529 SDend(sdfd);
1530 Hclose(fileid);
1531 string invalid_file_msg="HDF-EOS GDopen error for the file ";
1532 invalid_file_msg +=data_path;
1533 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1534 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1535 }
1536
1537 // Swath open
1538 swathfd = SWopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1539 if (-1 == swathfd) {
1540 SDend(sdfd);
1541 Hclose(fileid);
1542 GDclose(gridfd);
1543 string invalid_file_msg="HDF-EOS SWopen error for the file ";
1544 invalid_file_msg +=data_path;
1545 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1546 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1547 }
1548
1549
1550 // Here we will check if ECS_Metadata key if set. For DAP4's DMR,
1551 // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1552 // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1553 // This is one difference between DAP2 and DAP4 mapping. Since
1554 // people can use BES key to turn on the ECS metadata, so this is okay.
1555 // KY 2014-10-23
1556 bool ecs_metadata = true;
1557#if 0
1558 if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1559 || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1560#endif
1561 if((true == _disable_ecsmetadata_min)
1562 || (true == _disable_ecsmetadata_all))
1563 ecs_metadata = false;
1564
1565 try {
1566
1567 read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1568#if 0
1569 //read_das_use_eos2lib(*das, data_path,sdfd,fileid,gridfd,swathfd,true);
1570#endif
1571 Ancillary::read_ancillary_das(das, data_path);
1572
1573 // Pass file pointer(h4file, eosfile) from DAS to DDS
1574 read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1575 }
1576
1577 catch(...) {
1578 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1579 throw;
1580 }
1581 if(eosfile != nullptr)
1582 delete eosfile;
1583 GDclose(gridfd);
1584 SWclose(swathfd);
1585
1586#else
1587 try {
1588 read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1589 Ancillary::read_ancillary_das(das, data_path);
1590
1591 // Pass file pointer(h4file) from DAS to DDS.
1592 read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1593 }
1594 catch(...) {
1595 close_hdf4_fileid(sdfd,fileid,h4file);
1596 throw;
1597 //throw InternalErr(__FILE__,__LINE__,"read_dds_hdfsp error");
1598 }
1599#endif
1600 close_hdf4_fileid(sdfd,fileid,h4file);
1601
1602 }
1603 else {
1604 read_das(das, data_path);
1605 Ancillary::read_ancillary_das(das, data_path);
1606 read_dds(dds, data_path);
1607 }
1608
1609
1610 Ancillary::read_ancillary_dds(dds, data_path);
1611
1612 dds.transfer_attributes(&das);
1613
1614 }
1615
1616 catch (InternalErr & e) {
1617 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1618 __FILE__, __LINE__);
1619 }
1620 catch (Error & e) {
1621 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1622 __FILE__, __LINE__);
1623 }
1624 catch (...) {
1625 string s = "unknown exception caught building HDF4 DataDDS";
1626 throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1627 }
1628
1629#if 0
1630 //dds.print(cout);
1631 //dds.print_das(cout);
1632#endif
1633 //
1634 // Extract the DMR Response object - this holds the DMR used by the
1635 // other parts of the framework.
1636 BESResponseObject *response = dhi.response_handler->get_response_object();
1637 BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1638
1639 DMR *dmr = bes_dmr.get_dmr();
1640
1641 D4BaseTypeFactory MyD4TypeFactory;
1642 dmr->set_factory(&MyD4TypeFactory);
1643#if 0
1644 //dmr->set_factory(new D4BaseTypeFactory);
1645#endif
1646 dmr->build_using_dds(dds);
1647
1648#if 0
1649 //dmr->print(cout);
1650#endif
1651
1652 // Instead of fiddling with the internal storage of the DHI object,
1653 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1654 // methods to set the constraints. But, why? Ans: from Patrick is that
1655 // in the 'container' mode of BES each container can have a different
1656 // CE.
1657 bes_dmr.set_dap4_constraint(dhi);
1658 bes_dmr.set_dap4_function(dhi);
1659 dmr->set_factory(nullptr);
1660
1661 return true;
1662}
1663
1664bool HDF4RequestHandler::hdf4_build_dmr_with_IDs(BESDataHandlerInterface & dhi) {
1665
1666 BESStopWatch sw;
1667 if (BESDebug::IsSet(TIMING_LOG_KEY))
1668 sw.start("HDF4RequestHandler::hdf4_build_dmr_with_IDs", dhi.data[REQUEST_ID]);
1669
1670 // Because this code does not yet know how to build a DMR directly, use
1671 // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
1672 // First step, build the 'full DDS'
1673 string data_path = dhi.container->access();
1674
1675 BaseTypeFactory factory;
1676 DDS dds(&factory, name_path(data_path), "3.2");
1677 dds.filename(data_path);
1678
1679 DAS das;
1680
1681 int32 sdfd = -1;
1682 int32 fileid = -1;
1683
1684 HDFSP::File *h4file = nullptr;
1685
1686 // Obtain HDF4 file IDs
1687 //SDstart
1688 sdfd = SDstart (data_path.c_str(), DFACC_READ);
1689 if( -1 == sdfd){
1690 string invalid_file_msg="HDF4 SDstart error for the file ";
1691 invalid_file_msg +=data_path;
1692 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1693 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1694 }
1695
1696 // H open
1697 fileid = Hopen(data_path.c_str(), DFACC_READ,0);
1698 if (-1 == fileid) {
1699 SDend(sdfd);
1700 string invalid_file_msg="HDF4 SDstart error for the file ";
1701 invalid_file_msg +=data_path;
1702 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1703 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1704 }
1705
1706
1707#ifdef USE_HDFEOS2_LIB
1708
1709 int32 gridfd = -1;
1710 int32 swathfd = -1;
1711
1712 HDFEOS2::File *eosfile = nullptr;
1713 // Obtain HDF-EOS2 file IDs with the file open APIs.
1714 // Grid open
1715 gridfd = GDopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1716 if (-1 == gridfd) {
1717 SDend(sdfd);
1718 Hclose(fileid);
1719 string invalid_file_msg="HDF4 SDstart error for the file ";
1720 invalid_file_msg +=data_path;
1721 invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1722 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1723
1724 }
1725
1726 // Swath open
1727 swathfd = SWopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1728 if (-1 == swathfd) {
1729 SDend(sdfd);
1730 Hclose(fileid);
1731 GDclose(gridfd);
1732 string invalid_file_msg="HDF-EOS SWopen error for the file ";
1733 invalid_file_msg +=data_path;
1734 invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1735 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1736
1737 }
1738
1739 // Here we will check if ECS_Metadata key if set. For DAP4's DMR,
1740 // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1741 // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1742 // This is one difference between DAP2 and DAP4 mapping. Since
1743 // people can use BES key to turn on the ECS metadata, so this is okay.
1744 // KY 2014-10-23
1745 bool ecs_metadata = true;
1746#if 0
1747 if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1748 || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1749#endif
1750 if((true == _disable_ecsmetadata_min)
1751 || (true == _disable_ecsmetadata_all))
1752 ecs_metadata = false;
1753
1754 try {
1755
1756 read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1757#if 0
1758 //read_das_use_eos2lib(*das, data_path,sdfd,fileid,gridfd,swathfd,true);
1759#endif
1760 Ancillary::read_ancillary_das(das, data_path);
1761
1762 // Pass file pointer(h4file, eosfile) from DAS to DDS.
1763 read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1764 }
1765
1766 catch(...) {
1767 close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1768 throw;
1769 }
1770 if(eosfile != nullptr)
1771 delete eosfile;
1772
1773#else
1774 try {
1775 read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1776 Ancillary::read_ancillary_das(das, data_path);
1777
1778 // Pass file pointer(h4file) from DAS to DDS.
1779 read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1780 }
1781 catch(...) {
1782 close_hdf4_fileid(sdfd,fileid,h4file);
1783 throw;
1784 }
1785#endif
1786 if(h4file != nullptr)
1787 delete h4file;
1788
1789 Ancillary::read_ancillary_dds(dds, data_path);
1790
1791 dds.transfer_attributes(&das);
1792
1793
1794// File IDs are closed by the derived class.
1795#if 0
1796 if(true == usecf) {
1797#ifdef USE_HDFEOS2_LIB
1798 GDclose(gridfd);
1799 SWclose(swathfd);
1800
1801#endif
1802 SDend(sdfd);
1803 Hclose(fileid);
1804 }
1805
1806 //dds.print(cout);
1807 //dds.print_das(cout);
1808#endif
1809 // Extract the DMR Response object - this holds the DMR used by the
1810 // other parts of the framework.
1811 BESResponseObject *response = dhi.response_handler->get_response_object();
1812 BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1813
1814 // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1815 // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1816 // the BES built and add this one. The HDF4DMR object will close the open dataset when
1817 // the BES runs the DMR's destructor.
1818 DMR *dmr = bes_dmr.get_dmr();
1819 D4BaseTypeFactory MyD4TypeFactory;
1820 dmr->set_factory(&MyD4TypeFactory);
1821 dmr->build_using_dds(dds);
1822 auto hdf4_dmr = new HDF4DMR(dmr);
1823#ifdef USE_HDFEOS2_LIB
1824 hdf4_dmr->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
1825#else
1826 hdf4_dmr->setHDF4Dataset(sdfd,fileid);
1827#endif
1828 delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1829 bes_dmr.set_dmr(hdf4_dmr); // BESDMRResponse will delete hdf4_dmr
1830
1831 // Instead of fiddling with the internal storage of the DHI object,
1832 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1833 // methods to set the constraints. But, why? Ans: from Patrick is that
1834 // in the 'container' mode of BES each container can have a different
1835 // CE.
1836 bes_dmr.set_dap4_constraint(dhi);
1837 bes_dmr.set_dap4_function(dhi);
1838 hdf4_dmr->set_factory(nullptr);
1839
1840 return true;
1841
1842}
1843
1844bool HDF4RequestHandler::hdf4_build_help(BESDataHandlerInterface & dhi) {
1845 BESResponseObject *response = dhi.response_handler->get_response_object();
1846 auto info = dynamic_cast<BESInfo *> (response);
1847 if (!info)
1848 throw BESInternalError("cast error", __FILE__, __LINE__);
1849
1850 map < string, string > attrs;
1851 attrs["name"] = MODULE_NAME ;
1852 attrs["version"] = MODULE_VERSION ;
1853 list < string > services;
1854 BESServiceRegistry::TheRegistry()->services_handled(HDF4_NAME, services);
1855 if (services.empty() == false) {
1856 string handles = BESUtil::implode(services, ',');
1857 attrs["handles"] = handles;
1858 }
1859 info->begin_tag("module", &attrs);
1860 info->end_tag("module");
1861
1862 return true;
1863}
1864
1865bool HDF4RequestHandler::hdf4_build_version(BESDataHandlerInterface & dhi) {
1866 BESResponseObject *response = dhi.response_handler->get_response_object();
1867 auto info = dynamic_cast<BESVersionInfo *> (response);
1868 if (!info)
1869 throw BESInternalError("cast error", __FILE__, __LINE__);
1870
1871 info->add_module(MODULE_NAME, MODULE_VERSION);
1872
1873 return true;
1874}
1875
1876#ifdef USE_HDFEOS2_LIB
1877void close_fileid(int sdfd, int fileid,int gridfd, int swathfd,HDFSP:: File* h4file, HDFEOS2::File*eosfile) {
1878 if(h4file !=nullptr)
1879 delete h4file;
1880 if(sdfd != -1)
1881 SDend(sdfd);
1882 if(fileid != -1)
1883 Hclose(fileid);
1884
1885
1886 if(eosfile !=nullptr)
1887 delete eosfile;
1888 if(gridfd != -1)
1889 GDclose(gridfd);
1890 if(swathfd != -1)
1891 SWclose(swathfd);
1892
1893}
1894#endif
1895void close_hdf4_fileid(int sdfd, int fileid,HDFSP::File*h4file) {
1896
1897 if(h4file !=nullptr)
1898 delete h4file;
1899
1900 if(sdfd != -1)
1901 SDend(sdfd);
1902 if(fileid != -1)
1903 Hclose(fileid);
1904
1905}
1906
1907// Handling DAS and DDS cache. Currently we only apply the cache to special products(AIRS version 6 level 3 or 2).
1908// read or write DAS from/to a cache file.
1909bool rw_das_cache_file(const string & filename, DAS *das_ptr,bool w_flag) {
1910
1911 bool das_set_cache = false;
1912 FILE *das_file = nullptr;
1913
1914 if(false == w_flag) // open a cache file for reading.
1915 das_file = fopen(filename.c_str(),"r");
1916 else
1917 das_file = fopen(filename.c_str(),"w");
1918
1919 if(nullptr == das_file) {
1920 if(ENOENT == errno) {
1921 // Since the das service always tries to read the data from a cache and if the cache file doesn't exist,
1922 // it will generates a cache file, so here we set a flag to indicate if a cache file needs to be generated.
1923 if(false == w_flag) {
1924 BESDEBUG("h4", "DAS set cache key is true." << endl);
1925 das_set_cache = true;
1926 }
1927 }
1928 else
1929 throw BESInternalError( "An error occurred trying to open a das cache file " + get_errno(), __FILE__, __LINE__);
1930 }
1931 else {
1932
1933 int fd_das = fileno(das_file);
1934
1935 // Set a corresponding read(shared) or write(exclusive) lock.
1936 struct flock *l_das;
1937 if(false == w_flag)
1938 l_das = lock(F_RDLCK);
1939 else
1940 l_das = lock(F_WRLCK);
1941
1942 // Hold a lock.
1943 if(fcntl(fd_das,F_SETLKW,l_das) == -1) {
1944 fclose(das_file);
1945 ostringstream oss;
1946 oss << "cache process: " << l_das->l_pid << " triggered a locking error: " << get_errno();
1947 throw BESInternalError( oss.str(), __FILE__, __LINE__);
1948 }
1949
1950 if(false == w_flag){
1951 // Read DAS from a cache file
1952 BESDEBUG("h4", "Obtaining DAS from the cache file" << endl);
1953 try {
1954 das_ptr->parse(das_file);
1955 }
1956 catch(...) {
1957 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1958 fclose(das_file);
1959 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1960 }
1961 fclose(das_file);
1962 throw InternalErr(__FILE__,__LINE__,"Fail to parse the das from a das file.");
1963 }
1964 }
1965 else {
1966 // Write DAS to a cache file
1967 BESDEBUG("h4", "write DAS to a cache file" << endl);
1968 try {
1969 das_ptr->print(das_file);
1970 }
1971 catch(...) {
1972 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1973 fclose(das_file);
1974 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1975 }
1976 fclose(das_file);
1977 throw InternalErr(__FILE__,__LINE__,"Fail to generate a das cache file.");
1978 }
1979
1980 }
1981
1982 // Unlock the cache file
1983 if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1984 fclose(das_file);
1985 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1986 }
1987 fclose(das_file);
1988
1989 }
1990
1991 return das_set_cache;
1992
1993}
1994
1995// Read dds from a cache file.
1996bool r_dds_cache_file(const string & cache_filename, DDS *dds_ptr,const string & hdf4_filename) {
1997
1998 bool dds_set_cache = false;
1999 FILE *dds_file = nullptr;
2000 dds_file = fopen(cache_filename.c_str(),"rb");
2001
2002 if(nullptr == dds_file) {
2003 if(ENOENT == errno) {
2004 // Since the das service always tries to read the data from a cache and if the cache file doesn't exist,
2005 // it is supposed that the handler should generate a cache file,
2006 // so set a flag to indicate if a cache file needs to be generated.
2007 dds_set_cache = true;
2008 }
2009 else
2010 throw BESInternalError( "An error occurred trying to open a dds cache file " + get_errno(), __FILE__, __LINE__);
2011 }
2012 else {
2013
2014 int fd_dds = fileno(dds_file);
2015 struct flock *l_dds;
2016 l_dds = lock(F_RDLCK);
2017
2018 // hold a read(shared) lock to read dds from a file.
2019 if(fcntl(fd_dds,F_SETLKW,l_dds) == -1) {
2020 fclose(dds_file);
2021 ostringstream oss;
2022 oss << "cache process: " << l_dds->l_pid << " triggered a locking error: " << get_errno();
2023 throw BESInternalError( oss.str(), __FILE__, __LINE__);
2024 }
2025
2026 try {
2027 HDFCFUtil::read_sp_sds_dds_cache(dds_file,dds_ptr,cache_filename,hdf4_filename);
2028 }
2029 catch(...) {
2030 if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2031 fclose(dds_file);
2032 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2033 }
2034
2035 fclose(dds_file);
2036 throw InternalErr(__FILE__,__LINE__,"Fail to generate a dds cache file.");
2037 }
2038
2039 if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2040 fclose(dds_file);
2041 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2042 }
2043
2044 fclose(dds_file);
2045
2046 }
2047
2048 return dds_set_cache;
2049
2050}
2051
2052bool check_beskeys(const string & key) {
2053
2054 bool found = false;
2055 string doset ="";
2056 const string dosettrue ="true";
2057 const string dosetyes = "yes";
2058
2059 TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2060 if( true == found ) {
2061 doset = BESUtil::lowercase( doset ) ;
2062 if( dosettrue == doset || dosetyes == doset )
2063 return true;
2064 }
2065 return false;
2066
2067}
2068
2069bool get_beskeys(const string & key,string &key_value) {
2070
2071 bool found = false;
2072
2073 TheBESKeys::TheKeys()->get_value( key, key_value, found ) ;
2074 return found;
2075
2076}
2077
2078#if 0
2079int get_cachekey_int(const string key) {
2080
2081 bool found = false;
2082 int ret_value = 0;
2083 string doset ="";
2084
2085 TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2086 if( true == found ) {
2087 istringstream iss(doset);
2088 iss >>ret_value;
2089 }
2090 return ret_value;
2091
2092}
2093#endif
2094#if 0
2095void test_func(HDFSP::File**h4file) {
2096cerr<<"OK to pass pointer of a nullptr pointer "<<endl;
2097
2098}
2099#endif
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
Holds a DDS object within the BES.
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
Base exception class for the BES with basic string message.
Definition: BESError.h:59
informational response object
Definition: BESInfo.h:63
exception thrown if internal error encountered
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:254
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:617
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
void close_fileid(hid_t fid)
Definition: h5get.cc:434