50#include "config_hdf.h"
68#ifdef HAVE_SYS_PARAM_H
83#include <libdap/DDS.h>
84#include <libdap/DAS.h>
85#include <libdap/escaping.h>
86#include <libdap/parser.h>
87#include <libdap/InternalErr.h>
88#include <libdap/debug.h>
93#include "HDF4RequestHandler.h"
100#include "HDFSequence.h"
101#include "HDFTypeFactory.h"
108#define SIGNED_BYTE_TO_INT32 1
113#include "HDFUInt16.h"
115#include "HDFUInt32.h"
116#include "HDFFloat32.h"
117#include "HDFFloat64.h"
125#include "HDFSPArray_RealField.h"
126#include "HDFSPArrayGeoField.h"
127#include "HDFSPArrayMissField.h"
128#include "HDFSPArrayAddCVField.h"
129#include "HDFSPArray_VDField.h"
130#include "HDFCFStrField.h"
132#include "HDFCFUtil.h"
135#ifdef USE_HDFEOS2_LIB
137#include "HDFEOS2Array_RealField.h"
138#include "HDFEOS2ArrayGridGeoField.h"
139#include "HDFEOS2ArraySwathGeoField.h"
140#include "HDFEOS2ArrayMissField.h"
141#include "HDFEOS2ArraySwathDimMapField.h"
142#include "HDFEOS2ArraySwathGeoMultiDimMapField.h"
143#include "HDFEOS2ArraySwathGeoDimMapExtraField.h"
146#include "HDFEOS2HandleType.h"
154#define ATTR_STRING_QUOTE_FIX
156template <
class T >
string num2string(T n)
164void hdfeos_switch_to_buffer(
void *new_buffer);
165void hdfeos_delete_buffer(
void * buffer);
166void *hdfeos_string(
const char *yy_str);
168struct yy_buffer_state;
169yy_buffer_state *hdfeos_scan_string(
const char *str);
170extern int hdfeosparse(libdap::parser_arg *arg);
173void AddHDFAttr(DAS & das,
const string & varname,
174 const vector < hdf_attr > &hav);
175void AddHDFAttr(DAS & das,
const string & varname,
176 const vector < string > &anv);
178static void build_descriptions(DDS & dds, DAS & das,
179 const string & filename);
180static void SDS_descriptions(sds_map & map, DAS & das,
181 const string & filename);
182static void Vdata_descriptions(vd_map & map, DAS & das,
183 const string & filename);
184static void Vgroup_descriptions(DDS & dds, DAS & das,
185 const string & filename, sds_map & sdmap,
186 vd_map & vdmap, gr_map & grmap);
187static void GR_descriptions(gr_map & map, DAS & das,
188 const string & filename);
189static void FileAnnot_descriptions(DAS & das,
const string & filename);
190static vector < hdf_attr > Pals2Attrs(
const vector < hdf_palette > palv);
191static vector < hdf_attr > Dims2Attrs(
const hdf_dim dim);
193void read_das(DAS & das,
const string & filename);
194void read_dds(DDS & dds,
const string & filename);
198bool read_dds_hdfsp(DDS & dds,
const string & filename,int32 sdfd, int32 fileid,
const HDFSP::File*h4file);
199bool read_das_hdfsp(DAS & das,
const string & filename,int32 sdfd, int32 fileid,
HDFSP::File**h4filepptr);
202bool read_dds_hdfhybrid(DDS & dds,
const string & filename,int32 sdfd, int32 fileid,
const HDFSP::File*h4file);
203bool read_das_hdfhybrid(DAS & das,
const string & filename,int32 sdfd, int32 fileid,
HDFSP::File**h4filepptr);
210bool read_dds_special_1d_grid(DDS &dds,
const HDFSP::File *spf,
const string & filename,int32 sdfd,
bool can_cache);
211bool read_das_special_eos2(DAS &das,
const string & filename,int32 sdid, int32 fileid,
bool ecs_metadata,
HDFSP::File**h4filepptr);
212bool read_das_special_eos2_core(DAS &das,
const HDFSP::File *spf,
const string & filename,
bool ecs_metadata);
214void read_das_sds(DAS & das,
const string & filename,int32 sdfd,
bool ecs_metadata,
HDFSP::File**h4fileptr);
215void read_dds_sds(DDS &dds,
const string & filename,int32 sdfd,
HDFSP::File*h4file,
bool dds_set_cache);
217void change_das_mod08_scale_offset(DAS & das,
const HDFSP::File *spf);
220void read_dds_spfields(DDS &dds,
const string& filename,
const int sdfd,
const HDFSP::SDField *spsds, SPType sptype);
223void read_dds_spvdfields(DDS &dds,
const string& filename,
const int fileid,int32 vdref, int32 numrec,
HDFSP::VDField *spvd);
226int check_special_eosfile(
const string&filename,
string&grid_name,int32 sdfd);
230#ifdef USE_HDFEOS2_LIB
233void parse_ecs_metadata(DAS &das,
const string & metaname,
const string &metadata);
254int read_dds_hdfeos2(DDS & dds,
const string & filename,int32 sdfd, int32 gridfd, int32 swathfd,
const HDFSP::File*h4file,HDFEOS2::File*eosfile);
257int read_das_hdfeos2(DAS & das,
const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,
bool ecs_metadata,
HDFSP::File**h4filepptr,HDFEOS2::File**eosfilepptr);
261void read_dds_hdfeos2_grid_swath(DDS &dds,
const string&filename, HDFEOS2::Dataset *dataset,
int grid_or_swath,
bool ownll, SOType sotype,
bool multi_dmap,
262 int32 sdfd, int32 gridfd,int32 swathfd)
265 BESDEBUG(
"h4",
"Coming to read_dds_hdfeos2_grid_swath "<<endl);
267 if(grid_or_swath < 0 || grid_or_swath > 1)
268 throw InternalErr(__FILE__, __LINE__,
"The current type should be either grid or swath");
273 vector<struct dimmap_entry> dimmaps;
276 string modis_geofilename=
"";
277 bool geofile_has_dimmap =
false;
281 if (grid_or_swath == 1)
282 HDFCFUtil::obtain_dimmap_info(filename,dataset,dimmaps,modis_geofilename,geofile_has_dimmap);
287 const vector<HDFEOS2::Field*>& fields = dataset->getDataFields();
288 vector<HDFEOS2::Field*> all_fields = fields;
289 vector<HDFEOS2::Field*>::const_iterator it_f;
291 if(1 == grid_or_swath) {
292 auto sw =
static_cast<HDFEOS2::SwathDataset *
>(dataset);
293 const vector<HDFEOS2::Field*>geofields = sw->getGeoFields();
294 for (it_f = geofields.begin(); it_f != geofields.end(); it_f++)
295 all_fields.push_back(*it_f);
300 for(it_f = all_fields.begin(); it_f != all_fields.end(); it_f++)
302 BESDEBUG(
"h4",
"New field Name " <<(*it_f)->getNewName()<<endl);
304 BaseType *bt=
nullptr;
307 int fieldtype = (*it_f)->getFieldType();
311 bool changedtype =
false;
312 for (
auto const &cf:ctype_field_namelist){
313 if (cf == (*it_f)->getNewName()){
319 switch((*it_f)->getType())
322#define HANDLE_CASE2(tid, type) \
324 if(true == changedtype && fieldtype==0) \
325 bt = new (HDFFloat32) ((*it_f)->getNewName(), (dataset)->getName()); \
327 bt = new (type)((*it_f)->getNewName(), (dataset)->getName()); \
330#define HANDLE_CASE(tid, type)\
332 bt = new (type)((*it_f)->getNewName(), (dataset)->getName()); \
336 HANDLE_CASE(DFNT_CHAR8,
HDFStr)
337#ifndef SIGNED_BYTE_TO_INT32
338 HANDLE_CASE2(DFNT_INT8,
HDFByte)
342 HANDLE_CASE2(DFNT_UINT8,
HDFByte)
347 HANDLE_CASE2(DFNT_UCHAR8,
HDFByte)
349 throw InternalErr(__FILE__,__LINE__,
"unsupported data type.");
356 const vector<HDFEOS2::Dimension*>& dims= (*it_f)->getCorrectedDimensions();
357 vector<HDFEOS2::Dimension*>::const_iterator it_d;
360 if(DFNT_CHAR == (*it_f)->getType()) {
362 if((*it_f)->getRank() >1) {
364 HDFEOS2CFStrField * ar =
nullptr;
368 ar =
new HDFEOS2CFStrField(
369 (*it_f)->getRank() -1,
370 (grid_or_swath ==0)?gridfd:swathfd,
375 (*it_f)->getNewName(),
380 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFCFStr instance.");
382 for(it_d = dims.begin(); it_d != dims.begin()+dims.size()-1; it_d++){
383 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
394 HDFEOS2CFStr * sca_str =
nullptr;
397 sca_str =
new HDFEOS2CFStr(
398 (grid_or_swath ==0)?gridfd:swathfd,
402 (*it_f)->getNewName(),
407 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFCFStr instance.");
409 dds.add_var(sca_str);
417 else if(fieldtype == 0 || fieldtype == 3 || fieldtype == 5) {
420 if(grid_or_swath==0){
421 HDFEOS2Array_RealField *ar =
nullptr;
422 ar =
new HDFEOS2Array_RealField(
424 filename,
false,sdfd,gridfd,
425 dataset->getName(),
"", (*it_f)->getName(),
427 (*it_f)->getNewName(), bt);
428 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
429 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
435 else if(grid_or_swath==1){
437 string tempfieldname = (*it_f)->getName();
440 if((*it_f)->UseDimMap() &&
false == multi_dmap) {
443 if (!modis_geofilename.empty()) {
446 if (
true == HDFCFUtil::is_modis_dimmap_nonll_field(tempfieldname)) {
448 if(
false == geofile_has_dimmap) {
455 HDFEOS2Array_RealField *ar =
nullptr;
456 ar =
new HDFEOS2Array_RealField(
463 "MODIS_Swath_Type_GEO",
466 (*it_f)->getNewName(),
469 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
470 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
477 HDFEOS2ArraySwathDimMapField * ar =
nullptr;
484 ar =
new HDFEOS2ArraySwathDimMapField(
491 "MODIS_Swath_Type_GEO",
495 (*it_f)->getNewName(),
497 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
498 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
506 HDFEOS2ArraySwathDimMapField * ar =
nullptr;
512 ar =
new HDFEOS2ArraySwathDimMapField(
523 (*it_f)->getNewName(),
525 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
526 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
535 HDFEOS2ArraySwathDimMapField * ar =
nullptr;
536 ar =
new HDFEOS2ArraySwathDimMapField(
547 (*it_f)->getNewName(),
549 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
550 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
558 HDFEOS2Array_RealField * ar =
nullptr;
559 ar =
new HDFEOS2Array_RealField(
569 (*it_f)->getNewName(),
571 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
572 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
580 throw InternalErr(__FILE__, __LINE__,
"The current type should be either grid or swath");
585 else if(fieldtype == 1 || fieldtype == 2) {
588 if(grid_or_swath==0) {
590 HDFEOS2ArrayGridGeoField *ar =
nullptr;
592 bool ydimmajor = (*it_f)->getYDimMajor();
593 bool condenseddim = (*it_f)->getCondensedDim();
594 bool speciallon = (*it_f)->getSpecialLon();
595 int specialformat = (*it_f)->getSpecialLLFormat();
597 ar =
new HDFEOS2ArrayGridGeoField(
610 (*it_f)->getNewName(),
613 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
614 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
627 else if(grid_or_swath ==1) {
629 if(
true == multi_dmap) {
630 if((*it_f)->getRank() !=2)
631 throw InternalErr(__FILE__, __LINE__,
"For the multi-dimmap case, the field rank must be 2.");
632 int dim0size = (dims[0])->getSize();
633 int dim1size = (dims[1])->getSize();
634 int dim0offset = (*it_f)->getLLDim0Offset();
635 int dim1offset = (*it_f)->getLLDim1Offset();
636 int dim0inc = (*it_f)->getLLDim0Inc();
637 int dim1inc = (*it_f)->getLLDim1Inc();
640 fieldname =
"Latitude";
642 fieldname =
"Longitude";
644cerr<<
"hdfdesc: newfieldname is "<<(*it_f)->getNewName() <<endl;
645cerr<<
"hdfdesc: dim0size "<<dim0size <<endl;
646cerr<<
"hdfdesc: dim1size "<<dim1size <<endl;
647cerr<<
"hdfdesc: dim0offset "<<dim0offset <<endl;
648cerr<<
"hdfdesc: dim1offset "<<dim1offset <<endl;
649cerr<<
"hdfdesc: dim0inc "<<dim0inc <<endl;
650cerr<<
"hdfdesc: dim1inc "<<dim1inc <<endl;
653 HDFEOS2ArraySwathGeoMultiDimMapField * ar =
nullptr;
655 ar =
new HDFEOS2ArraySwathGeoMultiDimMapField(
667 (*it_f)->getNewName(),
670 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
671 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
680 if((*it_f)->UseDimMap()) {
683 if(!modis_geofilename.empty()) {
685 if (
false == geofile_has_dimmap) {
686 HDFEOS2ArraySwathGeoDimMapExtraField *ar =
nullptr;
687 ar =
new HDFEOS2ArraySwathGeoDimMapExtraField(
691 (*it_f)->getNewName(),
693 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
694 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
701 HDFEOS2ArraySwathDimMapField * ar =
nullptr;
707 ar =
new HDFEOS2ArraySwathDimMapField(
714 "MODIS_Swath_Type_GEO",
718 (*it_f)->getNewName(),
720 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
721 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
731 HDFEOS2ArraySwathDimMapField * ar =
nullptr;
732 ar =
new HDFEOS2ArraySwathDimMapField(
743 (*it_f)->getNewName(),
745 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
746 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
755 HDFEOS2ArraySwathGeoField * ar =
nullptr;
756 ar =
new HDFEOS2ArraySwathGeoField(
762 (*it_f)->getNewName(),
765 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
766 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
775 throw InternalErr(__FILE__, __LINE__,
"The current type should be either grid or swath");
781 else if(fieldtype == 4) {
783 if((*it_f)->getRank()!=1){
785 throw InternalErr(__FILE__, __LINE__,
"The rank of missing Z dimension field must be 1");
788 int nelem = ((*it_f)->getCorrectedDimensions()[0])->getSize();
789 HDFEOS2ArrayMissGeoField *ar =
nullptr;
790 ar =
new HDFEOS2ArrayMissGeoField(
793 (*it_f)->getNewName(),
796 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
797 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
805 throw InternalErr(__FILE__, __LINE__,
"Encounter unsupported datatype or The field type should be between 0 and 5. ");
815int read_dds_hdfeos2(DDS & dds,
const string & filename,int32 sdfd, int32 gridfd, int32 swathfd,
const HDFSP::File*spf,HDFEOS2::File*f)
818 BESDEBUG(
"h4",
"Coming to read_dds_hdfeos2 "<<endl);
821 dds.set_dataset_name(basename(filename));
829 if((basename(filename).size() >=5) && ((basename(filename)).compare(0,5,
"MERRA")==0))
832 if(
true == HDF4RequestHandler::get_enable_special_eos()) {
835 int ret_val = check_special_eosfile(filename,grid_name,sdfd);
846 if(2 == ret_val || 3 == ret_val) {
849 read_dds_special_1d_grid(dds,spf,filename,sdfd,
false);
867 const vector<HDFEOS2::GridDataset *>& grids = f->getGrids();
869 bool onelatlon = f->getOneLatLon();
872 SOType sotype = DEFAULT_CF_EQU;
875 for (
const auto &gd:grids){
878 ownll = onelatlon?onelatlon:gd->getLatLonFlag();
881 sotype = gd->getScaleType();
883 read_dds_hdfeos2_grid_swath(
884 dds, filename,
static_cast<HDFEOS2::Dataset*
>(gd), 0,ownll,sotype,
false,sdfd,gridfd,swathfd);
887 HDFCFUtil::add_cf_grid_cvs(dds,gd);
895 bool multi_dmap = f->getMultiDimMaps();
899 const vector<HDFEOS2::SwathDataset *>& swaths= f->getSwaths();
900 for (
const auto &swath:swaths) {
903 sotype = swath->getScaleType();
906 read_dds_hdfeos2_grid_swath(
907 dds, filename,
static_cast<HDFEOS2::Dataset*
>(swath), 1,
false,sotype,multi_dmap,sdfd,gridfd,swathfd);
918 ctype_field_namelist.clear();
926bool read_dds_hdfhybrid(DDS & dds,
const string & filename,int32 sdfd, int32 fileid,
const HDFSP::File*f)
930 BESDEBUG(
"h4",
"Coming to read_dds_hdfhybrid "<<endl);
933 dds.set_dataset_name(basename(filename));
939 for(
const auto &sdfield:spsds){
941 read_dds_spfields(dds,filename,sdfd,sdfield,f->
getSPType());
955 if(
true == HDF4RequestHandler::get_enable_hybrid_vdata()) {
958 if(
false == vd->getTreatAsAttrFlag()){
959 for(
const auto &vdf:vd->getFields()) {
961 read_dds_spvdfields(dds,filename,fileid, vd->getObjRef(),vdf->getNumRec(),vdf);
976bool read_das_hdfhybrid(DAS & das,
const string & filename,int32 sdfd, int32 fileid,
HDFSP::File**fpptr)
979 BESDEBUG(
"h4",
"Coming to read_das_hdfhybrid "<<endl);
990 throw InternalErr(e.
what());
999 for (
const auto &spfield:spsds) {
1002 AttrTable *at = das.get_table(spfield->getNewName());
1004 at = das.add_table(spfield->getNewName(),
new AttrTable);
1007 bool long_name_flag =
false;
1009 for (
const auto &attr:spfield->getAttributes()) {
1011 if(attr->getName() ==
"long_name") {
1012 long_name_flag =
true;
1017 if(
false == long_name_flag)
1018 at->append_attr(
"long_name",
"String", spfield->getName());
1021 for (
const auto& attr:spfield->getAttributes()) {
1024 if(attr->getType()==DFNT_UCHAR || attr->getType() == DFNT_CHAR){
1027 string tempstring2(attr->getValue().begin(),attr->getValue().end());
1028 string tempfinalstr= string(tempstring2.c_str());
1033 at->append_attr(attr->getNewName(),
"String" , (attr->getNewName()==
"fullpath")?tempfinalstr:
HDFCFUtil::escattr(tempfinalstr));
1036 for (
int loc=0; loc < attr->getCount() ; loc++) {
1046 int32 var_type = spfield->getType();
1060 if (
true == HDF4RequestHandler::get_enable_check_scale_offset_type() && at !=
nullptr)
1067 HDFCFUtil::handle_vdata_attrs_with_desc_key(f,das);
1079void read_dds_use_eos2lib(DDS & dds,
const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,
HDFSP::File*h4file,HDFEOS2::File*eosfile)
1082 BESDEBUG(
"h4",
"Coming to read_dds_use_eos2lib" <<endl);
1084 int ret_value = read_dds_hdfeos2(dds,filename,sdfd,gridfd,swathfd,h4file,eosfile);
1086 BESDEBUG(
"h4",
"ret_value of read_dds_hdfeos2 is "<<ret_value<<endl);
1104 if (0 == ret_value || 5 == ret_value || 4 == ret_value ) {
1105 if(
true == read_dds_hdfsp(dds, filename,sdfd,fileid,h4file))
1109 else if ( 1 == ret_value ) {
1112 if(
true ==read_dds_hdfhybrid(dds,filename,sdfd,fileid,h4file))
1122 if(
true == read_dds_hdfeos2(dds, filename)){
1125 if(
true == read_dds_hdfhybrid(dds,filename))
1130 if(read_dds_hdfsp(dds, filename)){
1137 read_dds(dds, filename);
1142void write_non_ecsmetadata_attrs(
HE2CF& cf) {
1144 cf.set_non_ecsmetadata_attrs();
1149void write_ecsmetadata(DAS& das,
HE2CF& cf,
const string& _meta)
1168 bool suffix_is_number =
true;
1169 vector<string> meta_nonum_names;
1170 vector<string> meta_nonum_data;
1172 string meta = cf.
get_metadata(_meta,suffix_is_number,meta_nonum_names, meta_nonum_data);
1174 if(
""==meta &&
true == suffix_is_number){
1178 BESDEBUG(
"h4",meta << endl);
1180 if (
false == suffix_is_number) {
1184 for (
unsigned int i = 0; i <meta_nonum_names.size(); i++)
1185 parse_ecs_metadata(das,meta_nonum_names[i],meta_nonum_data[i]);
1188 parse_ecs_metadata(das,_meta,meta);
1192void parse_ecs_metadata(DAS &das,
const string & metaname,
const string &metadata) {
1195 AttrTable *at = das.get_table(metaname);
1197 at = das.add_table(metaname,
new AttrTable);
1200 void *buf = hdfeos_string(metadata.c_str());
1203 if (hdfeosparse(&arg) != 0) {
1204 hdfeos_delete_buffer(buf);
1205 throw Error(
"HDF-EOS parse error while processing a " + metadata +
" HDFEOS attribute.");
1208 if (arg.status() ==
false) {
1209 (*BESLog::TheLog())<<
"HDF-EOS parse error while processing a "
1210 << metadata <<
" HDFEOS attribute. (2) " << endl;
1213 << arg.error()->get_error_message() << endl;
1217 hdfeos_delete_buffer(buf);
1221int read_das_hdfeos2(DAS & das,
const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,
1222 bool ecs_metadata,
HDFSP::File**spfpptr,HDFEOS2::File **fpptr)
1225 BESDEBUG(
"h4",
"Coming to read_das_hdfeos2 " << endl);
1237 if((basename(filename).size() >=5) && ((basename(filename)).compare(0,5,
"MERRA")==0)) {
1243 string check_enable_spec_eos_key=
"H4.EnableSpecialEOS";
1244 bool turn_on_enable_spec_eos_key=
false;
1245 turn_on_enable_spec_eos_key = HDFCFUtil::check_beskeys(check_enable_spec_eos_key);
1247 if(
true == HDF4RequestHandler::get_enable_special_eos()) {
1250 int ret_val = check_special_eosfile(filename,grid_name,sdfd);
1256 bool airs_l2_l3_v6 =
false;
1257 bool special_1d_grid =
false;
1260 if(2 == ret_val || 3 == ret_val) {
1270 throw InternalErr(e.
what());
1277 if(spf->Check_update_special(grid_name)==
true){
1279 special_1d_grid =
true;
1282 read_das_special_eos2_core(das,spf,filename,ecs_metadata);
1285 if(grid_name ==
"mod08") {
1286 change_das_mod08_scale_offset(das,spf);
1292 airs_l2_l3_v6 =
true;
1293 spf->Handle_AIRS_L23();
1294 read_das_special_eos2_core(das,spf,filename,ecs_metadata);
1305 if (
true == special_1d_grid ||
true == airs_l2_l3_v6) {
1313 HDFEOS2::File *f =
nullptr;
1317 f= HDFEOS2::File::Read(filename.c_str(),gridfd,swathfd);
1319 catch (HDFEOS2::Exception &e){
1325 if (!e.getFileType()){
1330 throw InternalErr(e.what());
1337 f->Prepare(filename.c_str());
1340 catch (HDFEOS2:: Exception &e) {
1343 throw InternalErr(e.what());
1352 cf.
open(filename,sdfd,fileid);
1359 SOType sotype = DEFAULT_CF_EQU;
1364 bool tempstrflag =
false;
1369 if (f->getSwaths().empty() ==
false) {
1370 string temp_fname = basename(filename);
1371 string temp_prod_prefix =
"AMSR_E";
1372 if ((temp_fname.size() > temp_prod_prefix.size()) &&
1373 (0 == (temp_fname.compare(0,temp_prod_prefix.size(),temp_prod_prefix)))) {
1380 bool gridname_change_valid_range =
false;
1381 if(1 == f->getGrids().size()) {
1382 string gridname = f->getGrids()[0]->getName();
1383 if (
"VIP_CMG_GRID" == gridname)
1384 gridname_change_valid_range =
true;
1388 bool is_modis_l1b =
false;
1391 for (
int i = 0; i<(
int) f->getSwaths().size(); i++) {
1392 const HDFEOS2::SwathDataset* swath = f->getSwaths()[i];
1393 string sname = swath->getName();
1394 if(
"MODIS_SWATH_Type_L1B" == sname){
1395 is_modis_l1b =
true;
1403 for (
int i = 0; i < (
int) f->getGrids().size(); i++) {
1405 const HDFEOS2::GridDataset* grid = f->getGrids()[i];
1406 string gname = grid->getName();
1407 sotype = grid->getScaleType();
1409 const vector<HDFEOS2::Field*>gfields = grid->getDataFields();
1411 for (
const auto &gf:gfields) {
1413 bool change_fvtype =
false;
1416 string fname = gf->getName();
1419 string newfname = gf->getNewName();
1421 BESDEBUG(
"h4",
"Original field name: " << fname << endl);
1422 BESDEBUG(
"h4",
"Corrected field name: " << newfname << endl);
1425 int fieldtype = gf->getFieldType();
1428 if (fieldtype == 0){
1431 if(gf->haveAddedFillValue()) {
1432 BESDEBUG(
"h4",
"Has an added fill value." << endl);
1433 float addedfillvalue =
1434 gf->getAddedFillValue();
1437 BESDEBUG(
"h4",
"Added fill value = "<<addedfillvalue);
1439 type, addedfillvalue);
1441 string coordinate = gf->getCoordinate();
1442 BESDEBUG(
"h4",
"Coordinate attribute: " << coordinate <<endl);
1443 if (coordinate !=
"")
1449 (
int)(f->getGrids().size()), fieldtype);
1461 if(fieldtype == 1 && (gf->getSpecialLLFormat())==3)
1466 if (fieldtype !=3) {
1467 string tempunits = gf->getUnits();
1469 "fieldtype " << fieldtype
1470 <<
" units" << tempunits
1477 AttrTable *at = das.get_table(newfname);
1480 if(sotype!=DEFAULT_CF_EQU && at!=
nullptr)
1482 bool has_Key_attr =
false;
1483 AttrTable::Attr_iter it = at->attr_begin();
1484 while (it!=at->attr_end())
1486 if(at->get_name(it)==
"Key")
1488 has_Key_attr =
true;
1494 if((
false == is_modis_l1b) && (
false == gridname_change_valid_range)&&(
false == has_Key_attr) &&
1495 (
true == HDF4RequestHandler::get_disable_scaleoffset_comp()))
1496 HDFCFUtil::handle_modis_special_attrs_disable_scale_comp(at,basename(filename),
true, newfname,sotype);
1500 bool changedtype = HDFCFUtil::change_data_type(das,sotype,newfname);
1503 if (
true == changedtype)
1504 ctype_field_namelist.push_back(newfname);
1506 HDFCFUtil::handle_modis_special_attrs(at,basename(filename),
true, newfname,sotype,gridname_change_valid_range,changedtype,change_fvtype);
1512 HDFCFUtil::handle_amsr_attrs(at);
1516 if((
false == change_fvtype) && at !=
nullptr) {
1517 int32 var_type = gf->getType();
1526 if (
true == HDF4RequestHandler::get_enable_check_scale_offset_type() && at!=
nullptr)
1533 HDFCFUtil::add_cf_grid_cv_attrs(das,grid);
1544 for (
int i = 0; i < (
int) f->getSwaths().size(); i++) {
1546 const HDFEOS2::SwathDataset* swath = f->getSwaths()[i];
1550 const vector<HDFEOS2::Field*> geofields = swath->getGeoFields();
1551 vector<HDFEOS2::Field*> all_fields = geofields;
1553 const vector<HDFEOS2::Field*> datafields = swath->getDataFields();
1554 for (
const auto &df:datafields)
1555 all_fields.push_back(df);
1557 auto total_geofields = (
int)(geofields.size());
1559 string gname = swath->getName();
1560 BESDEBUG(
"h4",
"Swath name: " << gname << endl);
1562 sotype = swath->getScaleType();
1565 int field_counter = 0;
1567 for (
const auto &af:all_fields)
1569 bool change_fvtype =
false;
1570 string fname = af->getName();
1571 string newfname = af->getNewName();
1572 BESDEBUG(
"h4",
"Original Field name: " << fname << endl);
1573 BESDEBUG(
"h4",
"Corrected Field name: " << newfname << endl);
1575 int fieldtype = af->getFieldType();
1576 if (fieldtype == 0){
1577 string coordinate = af->getCoordinate();
1578 BESDEBUG(
"h4",
"Coordinate attribute: " << coordinate <<endl);
1579 if (coordinate !=
"")
1586 if(fieldtype >0 && fieldtype !=3){
1587 string tempunits = af->getUnits();
1589 "fieldtype " << fieldtype
1590 <<
" units" << tempunits << endl);
1594 BESDEBUG(
"h4",
"Field Name: " << fname << endl);
1598 if (field_counter >=total_geofields) {
1599 if(af->haveAddedFillValue()){
1600 float addedfillvalue =
1601 af->getAddedFillValue();
1604 BESDEBUG(
"h4",
"Added fill value = "<<addedfillvalue);
1609 (
int)(f->getSwaths().size()), fieldtype);
1611 AttrTable *at = das.get_table(newfname);
1614 if(sotype!=DEFAULT_CF_EQU && at!=
nullptr)
1617 bool has_Key_attr =
false;
1618 AttrTable::Attr_iter it = at->attr_begin();
1619 while (it!=at->attr_end())
1621 if(at->get_name(it)==
"Key")
1623 has_Key_attr =
true;
1629 if((
false == is_modis_l1b) && (
false == gridname_change_valid_range) &&(
false == has_Key_attr) &&
1630 (
true == HDF4RequestHandler::get_disable_scaleoffset_comp()))
1631 HDFCFUtil::handle_modis_special_attrs_disable_scale_comp(at,basename(filename),
false,newfname,sotype);
1635 bool changedtype = HDFCFUtil::change_data_type(das,sotype,newfname);
1638 if (
true == changedtype)
1640 ctype_field_namelist.push_back(newfname);
1645 HDFCFUtil::handle_modis_special_attrs(at,basename(filename),
false,newfname,sotype,gridname_change_valid_range,changedtype,change_fvtype);
1651 HDFCFUtil::handle_amsr_attrs(at);
1655 if((
false == change_fvtype) && at !=
nullptr) {
1656 int32 var_type = af->getType();
1666 if (
true == HDF4RequestHandler::get_enable_check_scale_offset_type() && at !=
nullptr)
1680 if(ecs_metadata ==
true) {
1683 write_ecsmetadata(das,cf,
"CoreMetadata");
1685 write_ecsmetadata(das,cf,
"coremetadata");
1687 write_ecsmetadata(das,cf,
"ArchiveMetadata");
1689 write_ecsmetadata(das,cf,
"archivemetadata");
1691 write_ecsmetadata(das,cf,
"ProductMetadata");
1693 write_ecsmetadata(das,cf,
"productmetadata");
1697 if(
false == tempstrflag) {
1700 string check_disable_smetadata_key =
"H4.DisableStructMetaAttr";
1701 bool is_check_disable_smetadata =
false;
1702 is_check_disable_smetadata = HDFCFUtil::check_beskeys(check_disable_smetadata_key);
1705 if (
false == HDF4RequestHandler::get_disable_structmeta() ) {
1706 write_ecsmetadata(das, cf,
"StructMetadata");
1711 write_non_ecsmetadata_attrs(cf);
1723 string check_enable_sg_attr_key=
"H4.EnableSwathGridAttr";
1724 bool turn_on_enable_sg_attr_key=
false;
1725 turn_on_enable_sg_attr_key = HDFCFUtil::check_beskeys(check_enable_sg_attr_key);
1728 if(
true == HDF4RequestHandler::get_enable_swath_grid_attr()) {
1731 for (
int i = 0; i < (
int) f->getGrids().size(); i++) {
1734 HDFEOS2::GridDataset* grid = f->getGrids()[i];
1738 AttrTable*at =
nullptr;
1741 if(grid->getAttributes().empty() ==
false){
1742 at = das.get_table(gname);
1744 at = das.add_table(gname,
new AttrTable);
1749 const vector<HDFEOS2::Attribute *> grid_attrs = grid->getAttributes();
1750 for (
const auto &attr:grid_attrs) {
1752 int attr_type = attr->getType();
1755 if(attr_type==DFNT_UCHAR || attr_type == DFNT_CHAR){
1756 string tempstring2(attr->getValue().begin(),attr->getValue().end());
1757 auto tempfinalstr= string(tempstring2.c_str());
1766 for (
int loc=0; loc < attr->getCount() ; loc++) {
1777 for (
int i = 0; i < (
int) f->getSwaths().size(); i++) {
1779 const HDFEOS2::SwathDataset* swath = f->getSwaths()[i];
1780 string sname = swath->getName();
1781 AttrTable*at =
nullptr;
1784 if(swath->getAttributes().empty() ==
false) {
1785 at = das.get_table(sname);
1787 at = das.add_table(sname,
new AttrTable);
1791 const vector<HDFEOS2::Attribute *> swath_attrs = swath->getAttributes();
1792 for (
const auto &attr:swath_attrs) {
1794 int attr_type = attr->getType();
1797 if(attr_type==DFNT_UCHAR || attr_type == DFNT_CHAR){
1798 string tempstring2(attr->getValue().begin(),attr->getValue().end());
1799 string tempfinalstr= string(tempstring2.c_str());
1806 for (
int loc=0; loc < attr->getCount() ; loc++) {
1825void read_das_use_eos2lib(DAS & das,
const string & filename,
1826 int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,
bool ecs_metadata,
1827 HDFSP::File**h4filepptr,HDFEOS2::File**eosfilepptr)
1830 BESDEBUG(
"h4",
"Coming to read_das_use_eos2lib" << endl);
1832 int ret_value = read_das_hdfeos2(das,filename,sdfd,fileid, gridfd, swathfd,ecs_metadata,h4filepptr,eosfilepptr);
1834 BESDEBUG(
"h4",
"ret_value of read_das_hdfeos2 is "<<ret_value <<endl);
1850 if (ret_value == 4) {
1851 if(
true == read_das_special_eos2(das, filename,sdfd,fileid,ecs_metadata,h4filepptr))
1855 else if (ret_value == 2 || ret_value == 3) {
1858 else if (ret_value == 1) {
1861 if(
true == read_das_hdfhybrid(das,filename,sdfd,fileid,h4filepptr))
1865 if(
true == read_das_hdfsp(das, filename,sdfd, fileid,h4filepptr))
1873 if(
true == read_das_hdfeos2(das, filename)){
1876 if (
true == read_das_hdfhybrid(das,filename))
1881 if(
true == read_das_hdfsp(das, filename)){
1888 read_das(das, filename);
1895bool read_dds_hdfsp(DDS & dds,
const string & filename,int32 sdfd, int32 fileid,
const HDFSP::File*f)
1898 BESDEBUG(
"h4",
"Coming to read_dds_sp "<<endl);
1899 dds.set_dataset_name(basename(filename));
1905 for (
const auto& spf:spsds){
1910 if (
false == f->
Has_Dim_NoScale_Field() || (0 == spf->getFieldType()) || (
true == spf->IsDimScale())){
1912 read_dds_spfields(dds,filename,sdfd,spf,f->
getSPType());
1923 string check_ceres_vdata_key=
"H4.EnableCERESVdata";
1924 bool turn_on_ceres_vdata_key=
false;
1925 turn_on_ceres_vdata_key = HDFCFUtil::check_beskeys(check_ceres_vdata_key);
1928 bool output_vdata_flag =
true;
1929 if (
false == HDF4RequestHandler::get_enable_ceres_vdata() &&
1934 output_vdata_flag =
false;
1936 if(
true == output_vdata_flag) {
1938 if(!vd->getTreatAsAttrFlag()){
1939 for(
const auto &vdf:vd->getFields()) {
1941 read_dds_spvdfields(dds,filename,fileid,vd->getObjRef(),vdf->getNumRec(),vdf);
1957bool read_das_hdfsp(DAS & das,
const string & filename, int32 sdfd, int32 fileid,
HDFSP::File**fpptr)
1960 BESDEBUG(
"h4",
"Coming to read_das_sp "<<endl);
1972 throw InternalErr(e.
what());
1982 throw InternalErr(e.
what());
1989 string check_enable_vg_attr_key=
"H4.EnableVgroupAttr";
1990 bool turn_on_enable_vg_attr_key=
false;
1991 turn_on_enable_vg_attr_key = HDFCFUtil::check_beskeys(check_enable_vg_attr_key);
1995 if(
true == HDF4RequestHandler::get_enable_vgroup_attr()) {
1998 vector<HDFSP::AttrContainer *>vg_container = f->
getVgattrs();
2000 AttrTable *vgattr_at = das.get_table(vgattr_c->getName());
2002 vgattr_at = das.add_table(vgattr_c->getName(),
new AttrTable);
2004 for (
const auto &attr:vgattr_c->getAttributes()) {
2007 if(attr->getType()==DFNT_UCHAR || attr->getType() == DFNT_CHAR){
2008 string tempstring2(attr->getValue().begin(),attr->getValue().end());
2009 string tempfinalstr= string(tempstring2.c_str());
2012 vgattr_at->append_attr(attr->getNewName(),
"String" ,
HDFCFUtil::escattr(tempfinalstr));
2015 for (
int loc=0; loc < attr->getCount() ; loc++) {
2026 string core_metadata =
"";
2027 string archive_metadata =
"";
2028 string struct_metadata =
"";
2039 if((sp_attr->getName().compare(0, 12,
"CoreMetadata" )== 0) ||
2040 (sp_attr->getName().compare(0, 12,
"coremetadata" )== 0)){
2045 string tempstring(sp_attr->getValue().begin(),sp_attr->getValue().end());
2052 core_metadata.append(tempstring);
2054 else if((sp_attr->getName().compare(0, 15,
"ArchiveMetadata" )== 0) ||
2055 (sp_attr->getName().compare(0, 16,
"ArchivedMetadata")==0) ||
2056 (sp_attr->getName().compare(0, 15,
"archivemetadata" )== 0)){
2057 string tempstring(sp_attr->getValue().begin(),sp_attr->getValue().end());
2062 archive_metadata.append(tempstring);
2064 else if((sp_attr->getName().compare(0, 14,
"StructMetadata" )== 0) ||
2065 (sp_attr->getName().compare(0, 14,
"structmetadata" )== 0)){
2067 if (
false == HDF4RequestHandler::get_disable_structmeta()) {
2069 string tempstring(sp_attr->getValue().begin(),sp_attr->getValue().end());
2077 struct_metadata.append(tempstring);
2083 AttrTable *at = das.get_table(
"HDF_GLOBAL");
2085 at = das.add_table(
"HDF_GLOBAL",
new AttrTable);
2088 if(sp_attr->getType()==DFNT_UCHAR || sp_attr->getType() == DFNT_CHAR){
2089 string tempstring2(sp_attr->getValue().begin(),sp_attr->getValue().end());
2090 auto tempfinalstr= string(tempstring2.c_str());
2098 for (
int loc=0; loc < sp_attr->getCount() ; loc++) {
2099 string print_rep =
HDFCFUtil::print_attr(sp_attr->getType(), loc, (
void*) &(sp_attr->getValue()[0]));
2112 if(core_metadata.size() > 0){
2113 AttrTable *at = das.get_table(
"CoreMetadata");
2115 at = das.add_table(
"CoreMetadata",
new AttrTable);
2117 void *buf = hdfeos_string(core_metadata.c_str());
2120 if (hdfeosparse(&arg) != 0) {
2121 hdfeos_delete_buffer(buf);
2122 throw Error(
"Parse error while processing a CoreMetadata attribute.");
2126 if (arg.status() ==
false) {
2127 ERROR_LOG(
"Parse error while processing a CoreMetadata attribute. (2) " << endl);
2133 hdfeos_delete_buffer(buf);
2137 if(archive_metadata.size() > 0){
2138 AttrTable *at = das.get_table(
"ArchiveMetadata");
2140 at = das.add_table(
"ArchiveMetadata",
new AttrTable);
2142 void *buf = hdfeos_string(archive_metadata.c_str());
2144 if (hdfeosparse(&arg) != 0){
2145 hdfeos_delete_buffer(buf);
2146 throw Error(
"Parse error while processing an ArchiveMetadata attribute.");
2150 if (arg.status() ==
false) {
2151 ERROR_LOG(
"Parse error while processing an ArchiveMetadata attribute. (2) " << endl);
2157 hdfeos_delete_buffer(buf);
2161 if(struct_metadata.size() > 0){
2162 AttrTable *at = das.get_table(
"StructMetadata");
2164 at = das.add_table(
"StructMetadata",
new AttrTable);
2166 void *buf = hdfeos_string(struct_metadata.c_str());
2168 if (hdfeosparse(&arg) != 0){
2169 hdfeos_delete_buffer(buf);
2170 throw Error(
"Parse error while processing a StructMetadata attribute.");
2173 if (arg.status() ==
false) {
2174 ERROR_LOG(
"Parse error while processing a StructMetadata attribute. (2)" << endl);
2180 if (arg.status() ==
false) {
2181 (*BESLog::TheLog())<<
"Parse error while processing a StructMetadata attribute. (2)" << endl
2182 << arg.error()->get_error_message() << endl;
2186 hdfeos_delete_buffer(buf);
2195 bool global_slope_flag =
false;
2196 float intercept = 0.;
2197 bool global_intercept_flag =
false;
2201 HDFCFUtil::check_obpg_global_attrs(f,scaling,slope,global_slope_flag,intercept,global_intercept_flag);
2205 vector<HDFSP::SDField *>::const_iterator it_g;
2206 for(it_g = spsds.begin(); it_g != spsds.end(); it_g++){
2213 ((*it_g)->getFieldType() !=0)&&
2214 ((*it_g)->IsDimScale() ==
false))
2218 if (OTHERHDF == f->
getSPType() &&
true == (*it_g)->IsDimNoScale())
2221 AttrTable *at = das.get_table((*it_g)->getNewName());
2223 at = das.add_table((*it_g)->getNewName(),
new AttrTable);
2226 bool long_name_flag =
false;
2228 for(vector<HDFSP::Attribute *>::const_iterator i=(*it_g)->getAttributes().begin();
2229 i!=(*it_g)->getAttributes().end();i++) {
2230 if((*i)->getName() ==
"long_name") {
2231 long_name_flag =
true;
2236 if(
false == long_name_flag) {
2238 if((*it_g)->getFieldType() == 1)
2239 at->append_attr(
"standard_name",
"String",
"latitude");
2240 else if ((*it_g)->getFieldType() == 2) {
2241 at->append_attr(
"standard_name",
"String",
"longitude");
2247 if((*it_g)->getFieldType() == 1) {
2248 at->append_attr(
"long_name",
"String",
"latitude");
2249 at->append_attr(
"standard_name",
"String",
"latitude");
2252 else if ((*it_g)->getFieldType() == 2) {
2253 at->append_attr(
"long_name",
"String",
"longitude");
2254 at->append_attr(
"standard_name",
"String",
"longitude");
2259 at->append_attr(
"long_name",
"String", (*it_g)->getName());
2265 HDFCFUtil::add_obpg_special_attrs(f,das,*it_g,scaling,slope,global_slope_flag,intercept,global_intercept_flag);
2268 for(vector<HDFSP::Attribute *>::const_iterator i=(*it_g)->getAttributes().begin();i!=(*it_g)->getAttributes().end();i++) {
2271 if((*i)->getType()==DFNT_UCHAR || (*i)->getType() == DFNT_CHAR){
2272 string tempstring2((*i)->getValue().begin(),(*i)->getValue().end());
2273 string tempfinalstr= string(tempstring2.c_str());
2279 at->append_attr((*i)->getNewName(),
"String" ,((*i)->getNewName()==
"fullpath")?tempfinalstr:
HDFCFUtil::escattr(tempfinalstr));
2282 for (
int loc=0; loc < (*i)->getCount() ; loc++) {
2297 bool has_dim_info =
true;
2298 vector<HDFSP::AttrContainer *>::const_iterator it_end = (*it_g)->getDimInfo().end();
2299 if((*it_g)->getType() == DFNT_CHAR) {
2300 if((*it_g)->getRank() >1 && (*it_g)->getDimInfo().size() >1)
2301 it_end = (*it_g)->getDimInfo().begin()+(*it_g)->getDimInfo().size() -1;
2303 has_dim_info =
false;
2306 if(
true == has_dim_info) {
2308 for(vector<HDFSP::AttrContainer *>::const_iterator i=(*it_g)->getDimInfo().begin();i!=it_end;i++) {
2311 string attr_container_name = (*it_g)->getNewName() + (*i)->getName();
2312 AttrTable *dim_at = das.get_table(attr_container_name);
2314 dim_at = das.add_table(attr_container_name,
new AttrTable);
2316 for (
const auto &attr:(*i)->getAttributes()) {
2319 if(attr->getType()==DFNT_UCHAR || attr->getType() == DFNT_CHAR){
2320 string tempstring2(attr->getValue().begin(),attr->getValue().end());
2321 string tempfinalstr= string(tempstring2.c_str());
2324 dim_at->append_attr(attr->getNewName(),
"String" ,
HDFCFUtil::escattr(tempfinalstr));
2327 for (
int loc=0; loc < attr->getCount() ; loc++) {
2340 if((*it_g)->getFieldType() == 1){
2342 at->del_attr(
"units");
2343 at->append_attr(
"units",
"String",(*it_g)->getUnits());
2345 at->del_attr(
"valid_range");
2350 if((*it_g)->getFieldType() == 2){
2351 at->del_attr(
"units");
2352 at->append_attr(
"units",
"String",(*it_g)->getUnits());
2354 at->del_attr(
"valid_range");
2360 if((*it_g)->getFieldType() == 4){
2361 at->del_attr(
"units");
2362 at->append_attr(
"units",
"String",(*it_g)->getUnits());
2366 if((*it_g)->getFieldType() == 0){
2367 at->del_attr(
"coordinates");
2371 string coordinate = (*it_g)->getCoordinate();
2372 if (coordinate !=
"")
2373 at->append_attr(
"coordinates",
"String", coordinate);
2380 HDFCFUtil::handle_otherhdf_special_attrs(f,das);
2383 HDFCFUtil::add_missing_cf_attrs(f,das);
2387 for(it_g = spsds.begin(); it_g != spsds.end(); it_g++){
2389 AttrTable *at = das.get_table((*it_g)->getNewName());
2390 if (at !=
nullptr) {
2391 int32 var_type = (*it_g)->getType();
2405 if (
true == HDF4RequestHandler::get_enable_check_scale_offset_type() && at !=
nullptr)
2410 HDFCFUtil::handle_merra_ceres_attrs_with_bes_keys(f,das,filename);
2416 HDFCFUtil::handle_vdata_attrs_with_desc_key(f,das);
2433bool read_das_special_eos2(DAS &das,
const string& filename,int32 sdfd,int32 fileid,
bool ecs_metadata,
HDFSP::File**fpptr) {
2435 BESDEBUG(
"h4",
"Coming to read_das_special_eos2 " << endl);
2448 throw InternalErr(e.
what());
2458 throw InternalErr(e.
what());
2464 read_das_special_eos2_core(das, f, filename,ecs_metadata);
2476bool read_das_special_eos2_core(DAS &das,
const HDFSP::File* f,
const string& filename,
bool ecs_metadata) {
2478 BESDEBUG(
"h4",
"Coming to read_das_special_eos2_core "<<endl);
2480 string core_metadata =
"";
2481 string archive_metadata =
"";
2482 string struct_metadata =
"";
2491 if((attr->getName().compare(0, 12,
"CoreMetadata" )== 0) ||
2492 (attr->getName().compare(0, 12,
"coremetadata" )== 0)){
2494 if(ecs_metadata ==
true) {
2498 string tempstring(attr->getValue().begin(),attr->getValue().end());
2499 core_metadata.append(tempstring);
2502 else if((attr->getName().compare(0, 15,
"ArchiveMetadata" )== 0) ||
2503 (attr->getName().compare(0, 16,
"ArchivedMetadata")==0) ||
2504 (attr->getName().compare(0, 15,
"archivemetadata" )== 0)){
2505 if(ecs_metadata ==
true) {
2506 string tempstring(attr->getValue().begin(),attr->getValue().end());
2507 archive_metadata.append(tempstring);
2510 else if((attr->getName().compare(0, 14,
"StructMetadata" )== 0) ||
2511 (attr->getName().compare(0, 14,
"structmetadata" )== 0))
2515 AttrTable *at = das.get_table(
"HDF_GLOBAL");
2517 at = das.add_table(
"HDF_GLOBAL",
new AttrTable);
2520 if(attr->getType()==DFNT_UCHAR || attr->getType() == DFNT_CHAR){
2521 string tempstring2(attr->getValue().begin(),attr->getValue().end());
2522 auto tempfinalstr= string(tempstring2.c_str());
2530 for (
int loc=0; loc < attr->getCount() ; loc++) {
2543 if(ecs_metadata ==
true) {
2545 if(core_metadata.size() > 0){
2546 AttrTable *at = das.get_table(
"CoreMetadata");
2548 at = das.add_table(
"CoreMetadata",
new AttrTable);
2550 void *buf = hdfeos_string(core_metadata.c_str());
2553 if (hdfeosparse(&arg) != 0) {
2554 hdfeos_delete_buffer(buf);
2555 throw Error(
"Parse error while processing a CoreMetadata attribute.");
2559 if (arg.status() ==
false) {
2560 ERROR_LOG(
"Parse error while processing a CoreMetadata attribute. (2)" << endl);
2563 << arg.error()->get_error_message() << endl;
2567 hdfeos_delete_buffer(buf);
2572 if(archive_metadata.size() > 0){
2573 AttrTable *at = das.get_table(
"ArchiveMetadata");
2575 at = das.add_table(
"ArchiveMetadata",
new AttrTable);
2577 void *buf = hdfeos_string(archive_metadata.c_str());
2579 if (hdfeosparse(&arg) != 0) {
2580 hdfeos_delete_buffer(buf);
2581 throw Error(
"Parse error while processing an ArchiveMetadata attribute.");
2585 if (arg.status() ==
false) {
2586 ERROR_LOG(
"Parse error while processing an ArchiveMetadata attribute. (2)" << endl);
2590 hdfeos_delete_buffer(buf);
2597 for (
const auto &sdf:spsds){
2603 if(sdf->getFieldType() != 0){
2605 AttrTable *at = das.get_table(sdf->getNewName());
2607 at = das.add_table(sdf->getNewName(),
new AttrTable);
2609 string tempunits = sdf->getUnits();
2610 if(at->simple_find(
"units")== at->attr_end() && tempunits!=
"")
2611 at->append_attr(
"units",
"String" ,tempunits);
2612 if(sdf->getFieldType() == 1){
2613 if(at->simple_find(
"long_name")== at->attr_end())
2614 at->append_attr(
"long_name",
"String",
"Latitude");
2616 else if(sdf->getFieldType() == 2) {
2617 if(at->simple_find(
"long_name")== at->attr_end())
2618 at->append_attr(
"long_name",
"String",
"Longitude");
2622 AttrTable *at = das.get_table(sdf->getNewName());
2624 at = das.add_table(sdf->getNewName(),
new AttrTable);
2625 string tempcoors = sdf->getCoordinate();
2628 at->del_attr(
"coordinates");
2629 at->append_attr(
"coordinates",
"String",tempcoors);
2634 if(sdf->getAttributes().empty())
2637 AttrTable *at = das.get_table(sdf->getNewName());
2639 at = das.add_table(sdf->getNewName(),
new AttrTable);
2642 for (
const auto &attr:sdf->getAttributes()) {
2645 if(attr->getType()==DFNT_UCHAR || attr->getType() == DFNT_CHAR){
2646 string tempstring2(attr->getValue().begin(),attr->getValue().end());
2647 string tempfinalstr= string(tempstring2.c_str());
2653 at->append_attr(attr->getNewName(),
"String" ,(attr->getNewName()==
"fullpath")?tempfinalstr:
HDFCFUtil::escattr(tempfinalstr));
2656 for (
int loc=0; loc < attr->getCount() ; loc++) {
2666 HDFCFUtil::map_eos2_objects_attrs(das,filename);
2673void change_das_mod08_scale_offset(DAS &das,
const HDFSP::File *f) {
2679 for (
const auto &sdf:spsds) {
2680 if(sdf->getFieldType() == 0){
2681 AttrTable *at = das.get_table(sdf->getNewName());
2683 at = das.add_table(sdf->getNewName(),
new AttrTable);
2686 string add_offset_type;
2689 string add_offset_value=
"0";
2690 double orig_offset_value = 0;
2691 bool add_offset_modify =
false;
2697 AttrTable::Attr_iter it = at->attr_begin();
2698 while (it!=at->attr_end())
2700 if(at->get_name(it)==
"add_offset")
2702 add_offset_value = (*at->get_attr_vector(it)->begin());
2703 orig_offset_value = atof(add_offset_value.c_str());
2704 add_offset_type = at->get_type(it);
2705 if(add_offset_value ==
"0.0" || orig_offset_value == 0)
2706 add_offset_modify =
false;
2708 add_offset_modify =
true;
2716 if(
true == add_offset_modify) {
2719 string scale_factor_type;
2722 string scale_factor_value=
"";
2723 double orig_scale_value = 1;
2725 it = at->attr_begin();
2726 while (it!=at->attr_end())
2728 if(at->get_name(it)==
"scale_factor")
2730 scale_factor_value = (*at->get_attr_vector(it)->begin());
2731 orig_scale_value = atof(scale_factor_value.c_str());
2732 scale_factor_type = at->get_type(it);
2737 if(scale_factor_value.length() !=0) {
2738 double new_offset_value = -1 * orig_scale_value*orig_offset_value;
2740 at->del_attr(
"add_offset");
2752bool read_dds_special_1d_grid(DDS &dds,
const HDFSP::File* spf,
const string& filename, int32 sdid,
bool check_cache) {
2755 BESDEBUG(
"h4",
"Coming to read_dds_special_1d_grid "<<endl);
2756 bool dds_cache =
false;
2757 size_t total_bytes_dds_cache = 0;
2760 if(
true == check_cache) {
2762 total_bytes_dds_cache = HDFCFUtil::obtain_dds_cache_size(spf);
2763 BESDEBUG(
"h4",
"Total DDS cache file size is "<< total_bytes_dds_cache<<endl);
2764 if(total_bytes_dds_cache !=0)
2769 SPType sptype = OTHERHDF;
2770 const vector<HDFSP::SDField *>& spsds = spf->
getSD()->
getFields();
2773 for (
const auto &spsdsf:spsds) {
2775 BaseType *bt=
nullptr;
2776 switch(spsdsf->getType()) {
2777#define HANDLE_CASE(tid, type) \
2779 bt = new (type)(spsdsf->getNewName(),filename); \
2783 HANDLE_CASE(DFNT_CHAR,
HDFStr)
2784#ifndef SIGNED_BYTE_TO_INT32
2785 HANDLE_CASE(DFNT_INT8,
HDFByte)
2789 HANDLE_CASE(DFNT_UINT8,
HDFByte)
2794 HANDLE_CASE(DFNT_UCHAR8,
HDFByte)
2796 throw InternalErr(__FILE__,__LINE__,
"unsupported data type.");
2803 const vector<HDFSP::Dimension*>& dims= spsdsf->getDimensions();
2805 vector<HDFSP::Dimension*>::const_iterator it_d;
2808 if(DFNT_CHAR == spsdsf->getType()) {
2809 if(1 == spsdsf->getRank()) {
2814 spsdsf->getFieldRef(),
2817 spsdsf->getNewName(),
2823 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFCFStr instance.");
2825 dds.add_var(sca_str);
2835 spsdsf->getRank() -1 ,
2839 spsdsf->getFieldRef(),
2842 spsdsf->getNewName(),
2848 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFCFStrField instance.");
2851 for(it_d = dims.begin(); it_d != dims.begin()+dims.size()-1; it_d++)
2852 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
2863 if(spsdsf->getFieldType()!= 4) {
2868 vector<int32>dimsizes;
2870 dimsizes.resize(spsdsf->getRank());
2871 for(
int i = 0; i <spsdsf->getRank();i++)
2872 dimsizes[i] = (dims[i])->getSize();
2877 spsdsf->getFieldRef(),
2882 spsdsf->getNewName(),
2887 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFSPArray_RealField instance.");
2889 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
2890 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
2896 if(spsdsf->getRank()!=1){
2898 throw InternalErr(__FILE__, __LINE__,
"The rank of missing Z dimension field must be 1");
2900 int nelem = (spsdsf->getDimensions()[0])->getSize();
2908 spsdsf->getNewName(),
2913 throw InternalErr(__FILE__,__LINE__,
2914 "Unable to allocate the HDFSPArrayMissGeoField instance.");
2918 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
2919 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
2930 if(
true == dds_cache) {
2933 string md_cache_dir;
2934 string key =
"H4.Cache.metadata.path";
2941 string base_file_name = basename(filename);
2942 string dds_filename = md_cache_dir +
"/"+base_file_name +
"_dds";
2945 FILE* dds_file =fopen(dds_filename.c_str(),
"wb");
2946 if(
nullptr == dds_file) {
2947 string msg =
"Cannot create the cache file. " + dds_filename + get_errno();
2948 throw InternalErr(__FILE__,__LINE__,msg);
2950 int fd = fileno(dds_file);
2951 struct flock *l= lock(F_WRLCK);
2952 if (fcntl(fd, F_SETLKW, l) == -1) {
2954 string msg =
"Cannot hold the write lock for dds cached file "+ dds_filename;
2955 throw InternalErr (__FILE__, __LINE__,msg);
2959 HDFCFUtil::write_sp_sds_dds_cache(spf,dds_file,total_bytes_dds_cache,dds_filename);
2962 if (fcntl(fd, F_SETLK, lock(F_UNLCK)) == -1) {
2964 string msg =
"Cannot release the write lock for dds cached file "+ dds_filename;
2965 throw InternalErr (__FILE__, __LINE__,msg);
2969 throw InternalErr(__FILE__,__LINE__,
"Fail to generate a dds cache file.");
2971 if (fcntl(fd, F_SETLK, lock(F_UNLCK)) == -1) {
2973 string msg =
"Cannot release the write lock for dds cached file "+ dds_filename;
2974 throw InternalErr (__FILE__, __LINE__,msg);
2981 throw InternalErr (__FILE__, __LINE__,
2982 "DDS/DAS metadata cache path cannot be found when 'H4.EnableMetaDataCacheFile' key is set to be true.");
2991void read_dds_spfields(DDS &dds,
const string& filename,
const int sdfd,
const HDFSP::SDField *spsds, SPType sptype) {
2993 BESDEBUG(
"h4",
"Coming to read_dds_spfields "<<endl);
2996 if(OTHERHDF == sptype && (
true == spsds->
IsDimNoScale()))
2999 BaseType *bt=
nullptr;
3002#define HANDLE_CASE(tid, type) \
3004 bt = new (type)(spsds->getNewName(),filename); \
3008 HANDLE_CASE(DFNT_CHAR,
HDFStr)
3009#ifndef SIGNED_BYTE_TO_INT32
3010 HANDLE_CASE(DFNT_INT8,
HDFByte)
3016 HANDLE_CASE(DFNT_UINT8,
HDFByte);
3021 HANDLE_CASE(DFNT_UCHAR,
HDFByte);
3023 throw InternalErr(__FILE__,__LINE__,
"unsupported data type.");
3026 int fieldtype = spsds->getFieldType();
3032 vector<HDFSP::Dimension*>::const_iterator it_d;
3034 if(DFNT_CHAR == spsds->
getType()) {
3044 spsds->getFieldRef(),
3053 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFCFStr instance.");
3055 dds.add_var(sca_str);
3068 spsds->getFieldRef(),
3077 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFCFStrField instance.");
3080 for(it_d = dims.begin(); it_d != dims.begin()+dims.size()-1; it_d++)
3081 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
3090 else if(fieldtype == 0 || fieldtype == 3 ) {
3095 vector<int32>dimsizes;
3096 dimsizes.resize(spsds->
getRank());
3097 for(
int i = 0; i <spsds->
getRank();i++)
3098 dimsizes[i] = (dims[i])->getSize();
3104 spsds->getFieldRef(),
3114 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFSPArray_RealField instance.");
3117 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
3118 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
3125 else if(fieldtype == 1 || fieldtype == 2) {
3127 if(sptype == MODISARNSS || sptype == TRMML2_V7) {
3133 vector<int32>dimsizes;
3135 dimsizes.resize(spsds->
getRank());
3136 for(
int i = 0; i <spsds->
getRank();i++)
3137 dimsizes[i] = (dims[i])->getSize();
3143 spsds->getFieldRef(),
3153 throw InternalErr(__FILE__,__LINE__,
3154 "Unable to allocate the HDFSPArray_RealField instance.");
3158 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
3159 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
3174 spsds->getFieldRef(),
3184 throw InternalErr(__FILE__,__LINE__,
3185 "Unable to allocate the HDFSPArray_RealField instance.");
3188 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
3189 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
3197 else if(fieldtype == 4) {
3200 throw InternalErr(__FILE__, __LINE__,
"The rank of missing Z dimension field must be 1");
3215 throw InternalErr(__FILE__,__LINE__,
3216 "Unable to allocate the HDFSPArrayMissGeoField instance.");
3220 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
3221 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
3228 else if(fieldtype == 6) {
3232 throw InternalErr(__FILE__, __LINE__,
"The rank of added coordinate variable must be 1");
3248 throw InternalErr(__FILE__,__LINE__,
3249 "Unable to allocate the HDFSPArrayAddCVField instance.");
3253 for(it_d = dims.begin(); it_d != dims.end(); it_d++)
3254 ar->append_dim((*it_d)->getSize(), (*it_d)->getName());
3261 throw InternalErr(__FILE__, __LINE__,
"The field type should be one of 0,1,2,3,4 or 6.");
3269void read_dds_spvdfields(DDS &dds,
const string & filename,
const int fileid,int32 objref,int32 numrec,
HDFSP::VDField *spvd) {
3271 BESDEBUG(
"h4",
"Coming to read_dds_spvdfields "<<endl);
3274 BaseType *bt=
nullptr;
3276#define HANDLE_CASE(tid, type) \
3278 bt = new (type)(spvd->getNewName(),filename); \
3282 HANDLE_CASE(DFNT_CHAR8,
HDFStr)
3283#ifndef SIGNED_BYTE_TO_INT32
3284 HANDLE_CASE(DFNT_INT8,
HDFByte)
3288 HANDLE_CASE(DFNT_UINT8,
HDFByte)
3293 HANDLE_CASE(DFNT_UCHAR8,
HDFByte)
3297 throw InternalErr(__FILE__,__LINE__,
"unsupported data type.");
3304 if(DFNT_CHAR == spvd->
getType()) {
3325 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFCFStr instance.");
3327 dds.add_var(sca_str);
3351 throw InternalErr(__FILE__,__LINE__,
"Unable to allocate the HDFCFStrField instance.");
3354 string dimname0 =
"VDFDim0_"+spvd->
getNewName();
3355 ar->append_dim(numrec, dimname0);
3380 string dimname1 =
"VDFDim0_"+spvd->
getNewName();
3382 string dimname2 =
"VDFDim1_"+spvd->
getNewName();
3384 ar->append_dim(numrec,dimname1);
3388 ar->append_dim(numrec,dimname1);
3400int check_special_eosfile(
const string & filename,
string& grid_name,int32 sdfd) {
3404 int32 n_sd_attrs = 0;
3405 bool is_eos =
false;
3409 if (SDfileinfo (sdfd, &n_sds, &n_sd_attrs) == FAIL){
3410 throw InternalErr (__FILE__,__LINE__,
"SDfileinfo failed ");
3413 char attr_name[H4_MAX_NC_NAME];
3414 int32 attr_type = -1;
3415 int32 attr_count = -1;
3416 char structmdname[] =
"StructMetadata.0";
3419 for (
int attr_index = 0; attr_index < n_sd_attrs;attr_index++) {
3420 if(SDattrinfo(sdfd,attr_index,attr_name,&attr_type,&attr_count) == FAIL) {
3421 throw InternalErr (__FILE__,__LINE__,
"SDattrinfo failed ");
3424 if(strcmp(attr_name,structmdname)==0) {
3430 if(
true == is_eos) {
3434 int32 dim_sizes[H4_MAX_VAR_DIMS];
3435 int32 sds_dtype = 0;
3436 int32 n_sds_attrs = 0;
3437 char sds_name[H4_MAX_NC_NAME];
3438 char xdim_name[] =
"XDim";
3439 char ydim_name[] =
"YDim";
3441 string temp_grid_name1;
3442 string temp_grid_name2;
3443 bool xdim_is_cv_flag =
false;
3444 bool ydim_is_cv_flag =
false;
3448 for (sds_index = 0; sds_index < n_sds; sds_index++) {
3450 sds_id = SDselect (sdfd, sds_index);
3451 if (sds_id == FAIL) {
3452 throw InternalErr (__FILE__,__LINE__,
"SDselect failed ");
3456 int status = SDgetinfo (sds_id, sds_name, &sds_rank, dim_sizes,
3457 &sds_dtype, &n_sds_attrs);
3458 if (status == FAIL) {
3459 SDendaccess(sds_id);
3460 throw InternalErr (__FILE__,__LINE__,
"SDgetinfo failed ");
3466 if(strcmp(sds_name,xdim_name) == 0) {
3467 int32 sds_dimid = SDgetdimid(sds_id,0);
3468 if(sds_dimid == FAIL) {
3469 SDendaccess(sds_id);
3470 throw InternalErr (__FILE__,__LINE__,
"SDgetinfo failed ");
3472 char dim_name[H4_MAX_NC_NAME];
3475 int32 num_dim_attrs = 0;
3476 if(SDdiminfo(sds_dimid,dim_name,&dim_size,&dim_type,&num_dim_attrs) == FAIL) {
3477 SDendaccess(sds_id);
3478 throw InternalErr(__FILE__,__LINE__,
"SDdiminfo failed ");
3483 string tempdimname(dim_name);
3484 if(tempdimname.size() >=5) {
3485 if(tempdimname.compare(0,5,
"XDim:") == 0) {
3488 temp_grid_name1 = tempdimname.substr(5);
3489 xdim_is_cv_flag =
true;
3493 else if(
"XDim" == tempdimname)
3494 xdim_is_cv_flag =
true;
3499 if(strcmp(sds_name,ydim_name) == 0) {
3501 int32 sds_dimid = SDgetdimid(sds_id,0);
3502 if(sds_dimid == FAIL) {
3503 SDendaccess (sds_id);
3504 throw InternalErr (__FILE__,__LINE__,
"SDgetinfo failed ");
3506 char dim_name[H4_MAX_NC_NAME];
3509 int32 num_dim_attrs = 0;
3510 if(SDdiminfo(sds_dimid,dim_name,&dim_size,&dim_type,&num_dim_attrs) == FAIL) {
3511 SDendaccess(sds_id);
3512 throw InternalErr(__FILE__,__LINE__,
"SDdiminfo failed ");
3517 string tempdimname(dim_name);
3518 if(tempdimname.size() >=5) {
3519 if(tempdimname.compare(0,5,
"YDim:") == 0) {
3521 temp_grid_name2 = tempdimname.substr(5);
3522 ydim_is_cv_flag =
true;
3525 else if (
"YDim" == tempdimname)
3526 ydim_is_cv_flag =
true;
3531 SDendaccess(sds_id);
3532 if((
true == xdim_is_cv_flag) && (
true == ydim_is_cv_flag ))
3538 if ((temp_grid_name1 == temp_grid_name2) && (
true == xdim_is_cv_flag) && (
true == ydim_is_cv_flag)) {
3539 grid_name = temp_grid_name1;
3547 else if((basename(filename).size() >8) && (basename(filename).compare(0,4,
"AIRS") == 0)
3548 && ((basename(filename).find(
".L3.")!=string::npos) || (basename(filename).find(
".L2.")!=string::npos))){
3550 bool has_dimscale =
false;
3553 for (sds_index = 0; sds_index < n_sds; sds_index++) {
3555 sds_id = SDselect (sdfd, sds_index);
3556 if (sds_id == FAIL) {
3557 throw InternalErr (__FILE__,__LINE__,
"SDselect failed ");
3561 int status = SDgetinfo (sds_id, sds_name, &sds_rank, dim_sizes,
3562 &sds_dtype, &n_sds_attrs);
3563 if (status == FAIL) {
3564 SDendaccess(sds_id);
3565 throw InternalErr (__FILE__,__LINE__,
"SDgetinfo failed ");
3568 for (
int dim_index = 0; dim_index<sds_rank; dim_index++) {
3570 int32 sds_dimid = SDgetdimid(sds_id,dim_index);
3571 if(sds_dimid == FAIL) {
3572 SDendaccess(sds_id);
3573 throw InternalErr (__FILE__,__LINE__,
"SDgetinfo failed ");
3576 char dim_name[H4_MAX_NC_NAME];
3579 int32 num_dim_attrs = 0;
3580 if(SDdiminfo(sds_dimid,dim_name,&dim_size,&dim_type,&num_dim_attrs) == FAIL) {
3581 SDendaccess(sds_id);
3582 throw InternalErr(__FILE__,__LINE__,
"SDdiminfo failed ");
3586 has_dimscale =
true;
3591 SDendaccess(sds_id);
3592 if(
true == has_dimscale)
3597 if (
true == has_dimscale)
3608 bool has_dimscale =
true;
3609 bool is_grid =
false;
3612 for (sds_index = 0; sds_index < n_sds; sds_index++) {
3614 sds_id = SDselect (sdid, sds_index);
3615 if (sds_id == FAIL) {
3617 throw InternalErr (__FILE__,__LINE__,
"SDselect failed ");
3621 int status = SDgetinfo (sds_id, sds_name, &sds_rank, dim_sizes,
3622 &sds_dtype, &n_sds_attrs);
3623 if (status == FAIL) {
3624 SDendaccess(sds_id);
3626 throw InternalErr (__FILE__,__LINE__,
"SDgetinfo failed ");
3630 for (
int dim_index = 0; dim_index<sds_rank; dim_index++) {
3632 int32 sds_dimid = SDgetdimid(sds_id,dim_index);
3633 if(sds_dimid == FAIL) {
3634 SDendaccess(sds_id);
3636 throw InternalErr (__FILE__,__LINE__,
"SDgetinfo failed ");
3638 char dim_name[H4_MAX_NC_NAME];
3641 int32 num_dim_attrs = 0;
3642 if(SDdiminfo(sds_dimid,dim_name,&dim_size,&dim_type,&num_dim_attrs) == FAIL) {
3643 SDendaccess(sds_id);
3645 throw InternalErr(__FILE__,__LINE__,
"SDdiminfo failed ");
3649 has_dimscale =
false;
3653 SDendaccess(sds_id);
3655 if (
true == has_dimscale)
3666void read_das_sds(DAS & das,
const string & filename,int32 sdfd,
bool ecs_metadata,
HDFSP::File**h4fileptr) {
3671 spf->Handle_AIRS_L23();
3672 read_das_special_eos2_core(das,spf,filename,ecs_metadata);
3678 throw InternalErr(e.
what());
3687void read_dds_sds(DDS &dds,
const string & filename,int32 sdfd,
HDFSP::File*h4file,
bool dds_setcache) {
3690 dds.set_dataset_name(basename(filename));
3691 read_dds_special_1d_grid(dds,h4file,filename,sdfd,dds_setcache);
3696void read_dds(DDS & dds,
const string & filename)
3700 dds.set_dataset_name(basename(filename));
3701 build_descriptions(dds, das, filename);
3703 if (!dds.check_semantics()) {
3709void read_das(DAS & das,
const string & filename)
3713 dds.set_dataset_name(basename(filename));
3715 build_descriptions(dds, das, filename);
3717 if (!dds.check_semantics()) {
3725static void build_descriptions(DDS & dds, DAS & das,
3726 const string & filename)
3734 SDS_descriptions(sdsmap, das, filename);
3737 FileAnnot_descriptions(das, filename);
3740 Vdata_descriptions(vdatamap, das, filename);
3743 GR_descriptions(grmap, das, filename);
3746 Vgroup_descriptions(dds, das, filename, sdsmap, vdatamap, grmap);
3761 :
public binary_function < hdf_genvec &, hdf_attr, hdf_genvec & > {
3765 accum_attr(
const string & named):d_named(named) {
3771 BESDEBUG(
"h4",
"attr.name: " << attr.name << endl);
3772 if (attr.name.find(d_named) != string::npos) {
3775 stuff.assign(attr.values.data(), attr.values.size());
3776 cerr <<
"Attribute chunk: " << attr.name << endl;
3777 cerr << stuff << endl;
3779 accum.append(attr.values.number_type(), attr.values.data(),
3780 attr.values.size());
3789struct is_named:
public unary_function < hdf_attr, bool > {
3792 is_named(
const string & named):d_named(named) {
3795 bool operator() (
const hdf_attr & attr) {
3796 return (attr.name.find(d_named) != string::npos);
3801merge_split_eos_attributes(vector < hdf_attr > &attr_vec,
3802 const string & attr_name)
3805 if (count_if(attr_vec.begin(), attr_vec.end(), is_named(attr_name)) > 1) {
3809 attributes = accumulate(attr_vec.begin(), attr_vec.end(),
3810 attributes, accum_attr(attr_name));
3818 DBG(vector < string > s_m;
3819 attributes.print(s_m);
3820 cerr <<
"Accum struct MD: (" << s_m.size() <<
") "
3824 attr_vec.erase(remove_if(attr_vec.begin(), attr_vec.end(),
3825 is_named(attr_name)), attr_vec.end());
3829 merged_attr.name = attr_name;
3830 merged_attr.values = attributes;
3833 attr_vec.push_back(merged_attr);
3838static void SDS_descriptions(sds_map & map, DAS & das,
3839 const string & filename)
3843 sdsin.setmeta(
true);
3847 vector < hdf_attr > fileattrs;
3852 while (!sdsin.eos()) {
3855 sdi.in_vgroup =
false;
3856 map[sdi.sds.ref] = sdi;
3863 merge_split_eos_attributes(fileattrs,
"StructMetadata");
3864 merge_split_eos_attributes(fileattrs,
"CoreMetadata");
3865 merge_split_eos_attributes(fileattrs,
"ProductMetadata");
3866 merge_split_eos_attributes(fileattrs,
"ArchiveMetadata");
3867 merge_split_eos_attributes(fileattrs,
"coremetadata");
3868 merge_split_eos_attributes(fileattrs,
"productmetadata");
3871 AddHDFAttr(das,
string(
"HDF_GLOBAL"), fileattrs);
3873 vector < hdf_attr > dattrs;
3877 for (SDSI s = map.begin(); s != map.end(); ++s) {
3878 const hdf_sds *sds = &s->second.sds;
3879 AddHDFAttr(das, sds->name, sds->attrs);
3880 for (
int k = 0; k < (
int) sds->dims.size(); ++k) {
3881 dattrs = Dims2Attrs(sds->dims[k]);
3882 AddHDFAttr(das, sds->name +
"_dim_" + num2string(k), dattrs);
3891static void Vdata_descriptions(vd_map & map, DAS & das,
3892 const string & filename)
3898 while (!vdin.eos()) {
3901 vdi.in_vgroup =
false;
3902 map[vdi.vdata.ref] = vdi;
3907 vector < hdf_attr > dattrs;
3908 for (VDI s = map.begin(); s != map.end(); ++s) {
3910 AddHDFAttr(das, vd->name, vd->attrs);
3917static void Vgroup_descriptions(DDS & dds, DAS & das,
3918 const string & filename, sds_map & sdmap,
3919 vd_map & vdmap, gr_map & grmap)
3926 while (!vgin.eos()) {
3929 vgi.toplevel =
true;
3930 vgmap[vgi.vgroup.ref] = vgi;
3934 for (VGI v = vgmap.begin(); v != vgmap.end(); ++v) {
3938 AddHDFAttr(das, vg->name, vg->attrs);
3941 for (uint32 i = 0; i < vg->tags.size(); i++) {
3942 int32 tag = vg->tags[i];
3943 int32 ref = vg->refs[i];
3947 if (grmap.find(ref) != grmap.end())
3948 grmap[ref].in_vgroup =
true;
3950 vgmap[ref].toplevel =
false;
3953 vdmap[ref].in_vgroup =
true;
3956 sdmap[ref].in_vgroup =
true;
3959 ERROR_LOG(
"unknown tag: " << tag <<
" ref: " << ref << endl);
3970 for (VGI v = vgmap.begin(); v != vgmap.end(); ++v) {
3971 if (!v->second.toplevel)
3973 pbt = NewStructureFromVgroup(v->second.vgroup,
3974 vgmap, sdmap, vdmap,
3984 for (SDSI s = sdmap.begin(); s != sdmap.end(); ++s) {
3985 if (s->second.in_vgroup)
3987 if (s->second.sds.has_scale())
3988 pbt = NewGridFromSDS(s->second.sds, filename);
3990 pbt = NewArrayFromSDS(s->second.sds, filename);
3998 for (VDI v = vdmap.begin(); v != vdmap.end(); ++v) {
3999 if (v->second.in_vgroup)
4001 pbt = NewSequenceFromVdata(v->second.vdata, filename);
4008 for (GRI g = grmap.begin(); g != grmap.end(); ++g) {
4009 if (g->second.in_vgroup)
4011 pbt = NewArrayFromGR(g->second.gri, filename);
4019static void GR_descriptions(gr_map & map, DAS & das,
4020 const string & filename)
4027 vector < hdf_attr > fileattrs;
4032 while (!grin.eos()) {
4035 gri.in_vgroup =
false;
4036 map[gri.gri.ref] = gri;
4042 AddHDFAttr(das,
string(
"HDF_GLOBAL"), fileattrs);
4045 vector < hdf_attr > pattrs;
4046 for (GRI g = map.begin(); g != map.end(); ++g) {
4047 const hdf_gri *gri = &g->second.gri;
4049 AddHDFAttr(das, gri->name, gri->attrs);
4052 pattrs = Pals2Attrs(gri->palettes);
4053 AddHDFAttr(das, gri->name, pattrs);
4061static void FileAnnot_descriptions(DAS & das,
const string & filename)
4065 vector < string > fileannots;
4067 annotin >> fileannots;
4068 AddHDFAttr(das,
string(
"HDF_GLOBAL"), fileannots);
4075void AddHDFAttr(DAS & das,
const string & varname,
4076 const vector < hdf_attr > &hav)
4078 if (hav.size() == 0)
4082 string tempname = varname;
4083 AttrTable *atp = das.get_table(tempname);
4085 atp =
new AttrTable;
4086 atp = das.add_table(tempname, atp);
4089 vector < string > attv;
4091 for (
int i = 0; i < (
int) hav.size(); ++i) {
4093 attrtype = DAPTypeName(hav[i].values.number_type());
4095 attv = vector < string > ();
4096 hav[i].values.print(attv);
4099 for (
int j = 0; j < (
int) attv.size(); ++j) {
4101 string container_name = hav[i].name;
4102 if (container_name.find(
"StructMetadata") == 0
4103 || container_name.find(
"CoreMetadata") == 0
4104 || container_name.find(
"ProductMetadata") == 0
4105 || container_name.find(
"ArchiveMetadata") == 0
4106 || container_name.find(
"coremetadata") == 0
4107 || container_name.find(
"productmetadata") == 0) {
4108 string::size_type dotzero = container_name.find(
'.');
4109 if (dotzero != container_name.npos)
4110 container_name.erase(dotzero);
4113 AttrTable *at = das.get_table(container_name);
4115 at = das.add_table(container_name,
new AttrTable);
4118 void *buf = hdfeos_string(attv[j].c_str());
4136 if (hdfeosparse(&arg) != 0){
4137 hdfeos_delete_buffer(buf);
4138 throw Error(
"HDF-EOS parse error while processing a " + container_name +
" HDFEOS attribute.");
4142 if (arg.status() ==
false) {
4143 ERROR_LOG(
"HDF-EOS parse error while processing a "
4144 << container_name <<
" HDFEOS attribute. (2)" << endl);
4148 hdfeos_delete_buffer(buf);
4151 if (attrtype ==
"String")
4152#ifdef ATTR_STRING_QUOTE_FIX
4153 attv[j] = escattr(attv[j]);
4155 attv[j] =
"\"" + escattr(attv[j]) +
"\"";
4158 if (atp->append_attr(hav[i].name, attrtype, attv[j]) == 0)
4169void AddHDFAttr(DAS & das,
const string & varname,
4170 const vector < string > &anv)
4172 if (anv.size() == 0)
4177 AttrTable *atp = das.get_table(varname);
4179 atp =
new AttrTable;
4180 atp = das.add_table(varname, atp);
4184 for (
int i = 0; i < (
int) anv.size(); ++i) {
4185#ifdef ATTR_STRING_QUOTE_FIX
4186 an = escattr(anv[i]);
4188 an =
"\"" + escattr(anv[i]) +
"\"";
4190 if (atp->append_attr(
string(
"HDF_ANNOT"),
"String", an) == 0)
4200static vector < hdf_attr > Pals2Attrs(
const vector < hdf_palette > palv)
4202 vector < hdf_attr > pattrs;
4204 if (palv.size() != 0) {
4209 for (
int i = 0; i < (
int) palv.size(); ++i) {
4210 palname =
"hdf_palette_" + num2string(i);
4211 pattr.name = palname;
4212 pattr.values = palv[i].table;
4213 pattrs.push_back(pattr);
4214 pattr.name = palname +
"_ncomps";
4217 int32 *
>(&palv[i].ncomp), 1);
4218 pattrs.push_back(pattr);
4219 if (palv[i].name.length() != 0) {
4220 pattr.name = palname +
"_name";
4223 char *
>(palv[i].name.c_str()),
4224 palv[i].name.length());
4225 pattrs.push_back(pattr);
4234static vector < hdf_attr > Dims2Attrs(
const hdf_dim dim)
4236 vector < hdf_attr > dattrs;
4238 if (dim.name.length() != 0) {
4239 dattr.name =
"name";
4241 hdf_genvec(DFNT_CHAR,
const_cast < char *
>(dim.name.c_str()),
4243 dattrs.push_back(dattr);
4245 if (dim.label.length() != 0) {
4246 dattr.name =
"long_name";
4248 hdf_genvec(DFNT_CHAR,
const_cast < char *
>(dim.label.c_str()),
4249 dim.label.length());
4250 dattrs.push_back(dattr);
4252 if (dim.unit.length() != 0) {
4253 dattr.name =
"units";
4255 hdf_genvec(DFNT_CHAR,
const_cast < char *
>(dim.unit.c_str()),
4257 dattrs.push_back(dattr);
4259 if (dim.format.length() != 0) {
4260 dattr.name =
"format";
4262 hdf_genvec(DFNT_CHAR,
const_cast < char *
>(dim.format.c_str()),
4263 dim.format.length());
4264 dattrs.push_back(dattr);
This class provides a way to map HDF4 1-D character array to DAP Str for the CF option.
This class provides a way to map HDFEOS2 character >1D array to DAP Str array for the CF option.
This class provides a way to map HDFEOS2 1-D character array to DAP Str for the CF option.
const char * what() const override
Return exception message.
int32 getType() const
Get the data type of this field.
const std::string & getNewName() const
Get the CF name(special characters replaced by underscores) of this field.
int32 getRank() const
Get the dimension rank of this field.
const std::string & getName() const
Get the name of this field.
static File * Read(const char *path, int32 sdid, int32 fileid)
Retrieve SDS and Vdata information from the HDF4 file.
const std::vector< VDATA * > & getVDATAs() const
Public interface to Obtain Vdata.
const std::vector< AttrContainer * > & getVgattrs() const
Get attributes for all vgroups.
bool Has_Dim_NoScale_Field() const
This file has a field that is a SDS dimension but no dimension scale.
SD * getSD() const
Public interface to Obtain SD.
static File * Read_Hybrid(const char *path, int32 sdid, int32 fileid)
SPType getSPType() const
Obtain special HDF4 product type.
One instance of this class represents one SDS object.
const std::vector< Dimension * > & getCorrectedDimensions() const
Get the list of the corrected dimensions.
const std::vector< Dimension * > & getDimensions() const
Get the list of dimensions.
bool IsDimNoScale() const
Is this field a dimension without dimension scale(or empty[no data]dimension variable)
This class retrieves all SDS objects and SD file attributes.
const std::vector< Attribute * > & getAttributes() const
Public interface to obtain the SD(file) attributes.
const std::vector< SDField * > & getFields() const
Redundant member function.
One instance of this class represents one Vdata field.
int32 getFieldOrder() const
Get the order of this field.
bool open(const std::string &filename, const int sd_id, const int file_id)
openes \afilename HDF4 file.
string get_metadata(const std::string &metadataname, bool &suffix_is_num, std::vector< std::string > &non_num_names, std::vector< std::string > &non_num_data)
retrieves the merged metadata.
bool write_attribute(const std::string &gname, const std::string &fname, const std::string &newfname, int n_groups, int fieldtype)
void set_DAS(libdap::DAS *das)
sets DAS pointer so that we can bulid attribute tables.
bool close()
closes the opened file.
bool write_attribute_FillValue(const std::string &varname, int type, float val)
bool write_attribute_coordinates(const std::string &varname, std::string coord)
bool write_attribute_units(const std::string &varname, std::string units)
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
static std::string print_attr(int32, int, void *)
Print attribute values in string.
static std::string print_type(int32)
Print datatype in string.
static void correct_scale_offset_type(libdap::AttrTable *at)
static std::string get_CF_string(std::string s)
Change special characters to "_".
static std::string escattr(std::string s)
static void correct_fvalue_type(libdap::AttrTable *at, int32 dtype)