bes Updated for version 3.20.13
HE2CF.cc
1
2// This file is part of the hdf4 data handler for the OPeNDAP data server.
3// Copyright (c) 2010-2012 The HDF Group
4//
5// Author: Hyo-Kyung Lee <hyoklee@hdfgroup.org>
6//
7// This is free software; you can redistribute it and/or modify it under the
8// terms of the GNU Lesser General Public License as published by the Free
9// Software Foundation; either version 2.1 of the License, or (at your
10// option) any later version.
11//
12// This software is distributed in the hope that it will be useful, but
13// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
15// License for more details.
16//
17// You should have received a copy of the GNU Lesser General Public License
18// along with this software; if not, write to the Free Software Foundation,
19// Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20//
21// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23#include "HE2CF.h"
24#include <libdap/escaping.h>
25#include <libdap/InternalErr.h>
26#include "HDFCFUtil.h"
27#include <iomanip>
28
29using namespace libdap;
30using namespace std;
31
32// Private member functions
33bool
34HE2CF::get_vgroup_field_refids(const string& _gname,
35 int32* _ref_df,
36 int32* _ref_gf)
37{
38
39 int32 vrefid = Vfind(file_id, (char*)_gname.c_str());
40 if (FAIL == vrefid) {
41 Vend(file_id);
42 ostringstream error;
43 error <<"cannot obtain the reference number for vgroup "<<_gname;
44 throw_error(error.str());
45 return false;
46 }
47
48 int32 vgroup_id = Vattach(file_id, vrefid, "r");
49 if (FAIL == vgroup_id) {
50 Vend(file_id);
51 ostringstream error;
52 error <<"cannot obtain the group id for vgroup "<<_gname;
53 throw_error(error.str());
54 return false;
55 }
56
57 int32 npairs = Vntagrefs(vgroup_id);
58 int32 ref_df = -1;
59 int32 ref_gf = -1;
60
61 if(npairs < 0){
62 Vdetach(vgroup_id);
63 Vend(file_id);
64 ostringstream error;
65 error << "Got " << npairs
66 << " npairs for " << _gname;
67 throw_error(error.str());
68 return false;
69 }
70
71 for (int i = 0; i < npairs; ++i) {
72
73 int32 tag = 0;
74 int32 ref = 0;
75
76 if (Vgettagref(vgroup_id, i, &tag, &ref) < 0){
77
78 // Previously just output stderr, it doesn't throw an internal error.
79 // I believe this is wrong. Use DAP's internal error to throw an error.KY 2011-4-26
80 Vdetach(vgroup_id);
81 Vend(file_id);
82 ostringstream error;
83 error << "failed to get tag / ref";
84 throw_error(error.str());
85 return false;
86 }
87
88 if(Visvg(vgroup_id, ref)){
89
90 char cvgroup_name[VGNAMELENMAX*4]; // the child vgroup name
91 int32 istat = 0;
92 int32 vgroup_cid = 0; // Vgroup id
93
94 vgroup_cid = Vattach(file_id, ref,"r");
95 if (FAIL == vgroup_cid) {
96 Vdetach(vgroup_id);
97 Vend(file_id);
98 ostringstream error;
99 error << "cannot obtain the vgroup id";
100 throw_error(error.str());
101 return false;
102 }
103
104 istat = Vgetname(vgroup_cid,cvgroup_name);
105 if (FAIL == istat) {
106 Vdetach(vgroup_cid);
107 Vdetach(vgroup_id);
108 Vend(file_id);
109 ostringstream error;
110 error << "cannot obtain the vgroup id";
111 throw_error(error.str());
112 return false;
113 }
114
115 if(strncmp(cvgroup_name, "Data Fields", 11) == 0){
116 ref_df = ref;
117 }
118
119 if(strncmp(cvgroup_name, "Geolocation Fields", 18) == 0){
120 ref_gf = ref;
121 }
122
123 if (FAIL == Vdetach(vgroup_cid)) {
124 Vdetach(vgroup_id);
125 Vend(file_id);
126 ostringstream error;
127 error << "cannot close the vgroup "<< cvgroup_name <<"Successfully";
128 throw_error(error.str());
129 return false;
130 }
131
132 }
133 }
134 *_ref_df = ref_df;
135 *_ref_gf = ref_gf;
136
137 if (FAIL == Vdetach(vgroup_id)) {
138 Vend(file_id);
139 ostringstream error;
140 error << "cannot close the vgroup "<< _gname <<"Successfully";
141 throw_error(error.str());
142 return false;
143 }
144 return true;
145}
146
147bool
148HE2CF::open_sd(const string& _filename,const int sd_id_in)
149{
150 int32 num_datasets = -1;
151 sd_id = sd_id_in;
152 if(SDfileinfo(sd_id, &num_datasets, &num_global_attributes)
153 == FAIL){
154 if(file_id != -1)
155 Vend(file_id);
156 ostringstream error;
157 error << "Failed to call SDfileinfo() on "
158 << _filename
159 << " file.";
160 throw_error(error.str());
161 return false;
162 }
163 return true;
164}
165
166bool
167HE2CF::open_vgroup(const string& _filename,const int file_id_in)
168{
169
170 file_id = file_id_in;
171 if (Vstart(file_id) < 0){
172 ostringstream error;
173 error << "Failed to call Vstart on " << _filename << endl;
174 throw_error(error.str());
175 return false;
176 }
177 return true;
178}
179
180
181void HE2CF::set_DAS(DAS* _das)
182{
183 das = _das;
184}
185
186bool HE2CF::set_non_ecsmetadata_attrs() {
187
188 for(int i = 0; i < num_global_attributes; i++){
189
190 // H4_MAX_NC_NAME is from the user guide example. It's 256.
191 char temp_name[H4_MAX_NC_NAME];
192 int32 attr_type = 0;
193 int32 attr_count = 0;
194 if(SDattrinfo(sd_id, i, temp_name, &attr_type, &attr_count) == FAIL) {
195 Vend(file_id);
196 ostringstream error;
197 error << "Fail to obtain SDS global attribute info." << endl;
198 throw_error(error.str());
199 }
200
201 string attr_namestr(temp_name);
202 // Check if this attribute is an HDF-EOS2 metadata(coremeta etc. ) attribute
203 // If yes, ignore this attribute.
204 if (true == is_eosmetadata(attr_namestr))
205 continue;
206
207 // When DisableStructMetaAttr key is true, StructMetadata.0 is not in the
208 // ECS metadata list. So this routine will pick up this attribute and generate
209 // the DAP output here. Anyhow,
210 // StructMetadata attribute should not be generated here. We will turn it off.
211 if (attr_namestr.compare(0,14, "StructMetadata" )== 0)
212 continue;
213
214 // When DisableECSMetaDataAll key is true, All ECS attributes(Coremetadata etc.)
215 // should not be in the
216 // ECS metadata list. But this routine will pick up those attributes and generate
217 // the DAP output here. Anyhow,
218 // these attributes should not be generated here. We will turn it off.
219 if (attr_namestr.compare(0,12, "CoreMetadata" )== 0)
220 continue;
221 if (attr_namestr.compare(0,12, "coremetadata" )== 0)
222 continue;
223 if (attr_namestr.compare(0,15, "ArchiveMetadata" )== 0)
224 continue;
225 if (attr_namestr.compare(0,15, "archivemetadata" )== 0)
226 continue;
227 if (attr_namestr.compare(0,15, "Productmetadata" )== 0)
228 continue;
229 if (attr_namestr.compare(0,15, "productmetadata" )== 0)
230 continue;
231
232 // USE VECTOR
233 vector<char>attr_data;
234 attr_data.resize((attr_count+1) *DFKNTsize(attr_type));
235
236 if(SDreadattr(sd_id, i, attr_data.data()) == FAIL){
237 Vend(file_id);
238 //delete[] attr_data;
239 ostringstream error;
240 error << "Fail to read SDS global attributes" << endl;
241 throw_error(error.str());
242
243 }
244
245 // Handle character type attribute as a string.
246 if (attr_type == DFNT_CHAR || attr_type == DFNT_UCHAR) {
247 attr_data[attr_count] = '\0';
248 attr_count = 1;
249 }
250
251 AttrTable *at = das->get_table("HDF_GLOBAL");
252 if (!at)
253 at = das->add_table("HDF_GLOBAL", new AttrTable);
254
255 attr_namestr = HDFCFUtil::get_CF_string(attr_namestr);
256
257#if 0
258 if(attr_type == DFNT_UCHAR || attr_type == DFNT_CHAR){
259 string tempstring2(attr_data);
260 string tempfinalstr= string(tempstring2.c_str());
261
262 // Using the customized escattr. Don't escape \n,\r and \t. KY 2013-10-14
263 //tempfinalstr=escattr(tempfinalstr);
264 at->append_attr(attr_namestr, "String" , HDFCFUtil::escattr(tempfinalstr));
265 }
266
267#endif
268
269 for (int loc=0; loc < attr_count ; loc++) {
270 string print_rep = HDFCFUtil::print_attr(attr_type, loc, (void*)attr_data.data() );
271 at->append_attr(attr_namestr, HDFCFUtil::print_type(attr_type), print_rep);
272 }
273
274 }
275
276 return true;
277}
278
279
280
281// Combine ECS metadata coremetadata.0,coremetadata.1 etc. into one string.
282bool HE2CF::set_metadata(const string& metadata_basename,vector<string>& non_number_names, vector<string>& no_num_data)
283{
284 bool suffix_is_num_or_null = true;
285 metadata.clear();
286
287 // Metadata like coremetadata,coremetadata.0,coremetadata.1
288 list<string> one_dot_names;
289
290 // Metadata like coremetadata, coremetadata.0, coremetadata.0.1
291 list<string> two_dots_names;
292
293 // Metadata with the suffix as .s, .t, like productmetadta.s, productmetadata.t
294 // is non_number_names passed from the function
295
296 int list_flag = -1;
297
298 for(int i = 0; i < num_global_attributes; i++){
299
300 // H4_MAX_NC_NAME is from the user guide example. It's 256.
301 char temp_name[H4_MAX_NC_NAME];
302 int32 attr_type=0;
303 int32 attr_count = 0;
304 if(SDattrinfo(sd_id, i, temp_name, &attr_type, &attr_count) == FAIL) {
305 Vend(file_id);
306 ostringstream error;
307 error << "Fail to obtain SDS global attribute info." << endl;
308 throw_error(error.str());
309 }
310
311 string temp_name_str(temp_name);
312
313 // Find the basename, arrange the metadata name list.
314 if(temp_name_str.find(metadata_basename)==0) {
315 arrange_list(one_dot_names,two_dots_names,non_number_names,temp_name_str,list_flag);
316 }
317 }
318
319 list<string>::const_iterator lit;
320
321 // list_flag = 0, no suffix
322 // list_flag = 1, only .0, coremetadata.0
323 // list_flag = 2, coremetadata.0, coremetadata.1 etc
324 // list_flag = 3, coremeatadata.0, coremetadata.0.1 etc
325//cerr<<"list_flag "<<list_flag <<endl;
326 if ( list_flag >= 0 && list_flag <=2) {
327 for (lit = one_dot_names.begin();lit!=one_dot_names.end();++lit) {
328 set_eosmetadata_namelist(*lit);
329 string cur_data;
330 obtain_SD_attr_value(*lit,cur_data);
331//cerr<<"metadata "<<cur_data <<endl;
332 metadata.append(cur_data);
333 }
334 }
335
336 if( 3== list_flag) {
337 for (lit = two_dots_names.begin();lit!=two_dots_names.end();++lit){
338 set_eosmetadata_namelist(*lit);
339 string cur_data;
340 obtain_SD_attr_value(*lit,cur_data);
341 metadata.append(cur_data);
342 }
343 }
344
345 if(non_number_names.size() >0) {
346 suffix_is_num_or_null = false;
347 no_num_data.resize(non_number_names.size());
348 }
349
350 for (unsigned int i =0; i<non_number_names.size();i++) {
351 set_eosmetadata_namelist(non_number_names[i]);
352 obtain_SD_attr_value(non_number_names[i],no_num_data[i]);
353 }
354
355 return suffix_is_num_or_null;
356
357}
358
359// This routine will generate three ECS metadata lists. Note in theory list sl1 and sl2 should be sorted.
360// Since the ECS metadata is always written(sorted) in increasing numeric order, we don't perform this now.
361// Should watch if there are any outliers. KY 2012-08-31
362void HE2CF::arrange_list(list<string> & sl1, list<string>&sl2,vector<string>&v1,string name,int& flag) {
363
364 // No dot in the ECS name
365 if(name.find(".") == string::npos) {
366 sl1.push_front(name);
367 sl2.push_front(name);
368 flag = 0;
369 }
370 else if (name.find_first_of(".") == name.find_last_of(".")) {
371
372 size_t dot_pos = name.find_first_of(".");
373
374 if((dot_pos+1)==name.size())
375 throw InternalErr(__FILE__, __LINE__,"Should have characters or numbers after ." );
376
377 string str_after_dot = name.substr(dot_pos+1);
378 stringstream sstr(str_after_dot);
379
380 int number_after_dot = 0;
381 sstr >> number_after_dot;
382
383 // No dot after ECS metadata
384 if (!sstr)
385 v1.push_back(name);
386 // .0 after the main name of ECS metadata
387 else if(0 == number_after_dot) {
388 sl1.push_back(name);
389 sl2.push_back(name);
390 // For only .0 case, set flag to 1.
391 if(flag!=1)
392 flag =1;
393 }
394 else {// .1 or .2 etc. after the main ECS metadata name
395 sl1.push_back(name);
396 if (3 == flag)
397 throw InternalErr(__FILE__, __LINE__,
398 "ecs metadata suffix .1 and .0.1 cannot exist at the same file" );
399 if (flag !=2)
400 flag = 2;
401 }
402 }
403 else {// We don't distinguish if .0.1 and .0.0.1 will appear.
404 // have two dots in the ECS name.
405 sl2.push_back(name);
406 if (2 == flag)
407 throw InternalErr(__FILE__, __LINE__,"ecs metadata suffix .1 and .0.1 cannot exist at the same file" );
408 if (flag !=3)
409 flag = 3;
410 }
411
412}
413
414// Obtain SD attribute value
415void HE2CF::obtain_SD_attr_value(const string& attrname, string &cur_data) {
416
417 int32 sds_index = SDfindattr(sd_id, attrname.c_str());
418 if(sds_index == FAIL){
419 Vend(file_id);
420 ostringstream error;
421 error << "Failed to obtain the SDS global attribute" << attrname << endl;
422 throw InternalErr(__FILE__, __LINE__,error.str());
423 }
424
425 // H4_MAX_NC_NAME is from the user guide example. It's 256.
426 char temp_name[H4_MAX_NC_NAME];
427 int32 type = 0;
428 int32 count = 0;
429
430 if(SDattrinfo(sd_id, sds_index, temp_name, &type, &count) == FAIL) {
431 Vend(file_id);
432 ostringstream error;
433 error << "Failed to obtain the SDS global attribute" << attrname << "information" << endl;
434 throw InternalErr(__FILE__, __LINE__,error.str());
435 }
436
437 vector<char> attrvalue;
438 attrvalue.resize((count+1)*DFKNTsize(type));
439
440 if(SDreadattr(sd_id, sds_index, attrvalue.data()) == FAIL){
441 Vend(file_id);
442 ostringstream error;
443 error << "Failed to read the SDS global attribute" << attrname << endl;
444 throw InternalErr(__FILE__, __LINE__,error.str());
445
446 }
447 // Remove the last nullptr character
448 // string temp_data(attrvalue.begin(),attrvalue.end()-1);
449 // cur_data = temp_data;
450
451 if(attrvalue[count] != '\0')
452 throw InternalErr(__FILE__,__LINE__,"the last character of the attribute buffer should be nullptr");
453
454 // No need to escape the special characters since they are ECS metadata. Will see. KY 2013-10-14
455 cur_data.resize(attrvalue.size()-1);
456 copy(attrvalue.begin(),attrvalue.end()-1,cur_data.begin());
457}
458
459
460bool HE2CF::set_vgroup_map(int32 _refid,bool isgeo)
461{
462 // Clear existing maps first.
463 // Note: It should only clear the corresponding groups: Geolocation or Data.
464 if(false == isgeo) {
465 vg_dsd_map.clear();
466 vg_dvd_map.clear();
467 }
468 else {
469 vg_gsd_map.clear();
470 vg_gvd_map.clear();
471 }
472
473 int32 vgroup_id = Vattach(file_id, _refid, "r");
474 if (FAIL == vgroup_id) {
475 Vend(file_id);
476 ostringstream error;
477 error << "Fail to attach the vgroup " ;
478 throw_error(error.str());
479 return false;
480 }
481
482 int32 npairs = Vntagrefs(vgroup_id);
483 if (FAIL == npairs) {
484 Vdetach(vgroup_id);
485 Vend(file_id);
486 ostringstream error;
487 error << "Fail to obtain the number of objects in a group " ;
488 throw_error(error.str());
489 return false;
490 }
491
492 for (int i = 0; i < npairs; ++i) {
493
494 int32 tag2 = 0;
495 int32 ref2 = 0;
496 char buf[H4_MAX_NC_NAME];
497
498 if (Vgettagref(vgroup_id, i, &tag2, &ref2) < 0){
499 Vdetach(vgroup_id);
500 Vend(file_id);
501 ostringstream error;
502 error << "Vgettagref failed for vgroup_id=." << vgroup_id;
503 throw_error(error.str());
504 return false;
505 }
506
507 if(tag2 == DFTAG_NDG){
508
509 int32 sds_index = SDreftoindex(sd_id, ref2); // index
510 if (FAIL == sds_index) {
511 Vdetach(vgroup_id);
512 Vend(file_id);
513 ostringstream error;
514 error << "Cannot obtain the SDS index ";
515 throw_error(error.str());
516 return false;
517 }
518
519 int32 sds_id = SDselect(sd_id, sds_index); // sds_id
520 if (FAIL == sds_id) {
521 Vdetach(vgroup_id);
522 Vend(file_id);
523 ostringstream error;
524 error << "Cannot obtain the SDS ID ";
525 throw_error(error.str());
526 return false;
527 }
528
529 int32 rank;
530 int32 dimsizes[H4_MAX_VAR_DIMS];
531 int32 datatype;
532 int32 num_attrs;
533
534 if(FAIL == SDgetinfo(sds_id, buf, &rank, dimsizes, &datatype, &num_attrs)) {
535 Vdetach(vgroup_id);
536 Vend(file_id);
537 ostringstream error;
538 error << "Cannot obtain the SDS info.";
539 throw_error(error.str());
540 return false;
541 }
542 if(false == isgeo)
543 vg_dsd_map[string(buf)] = sds_id;
544 else
545 vg_gsd_map[string(buf)] = sds_id;
546
547 }
548
549 if(tag2 == DFTAG_VH){
550
551 int vid;
552 if ((vid = VSattach(file_id, ref2, "r")) < 0) {
553
554 Vdetach(vgroup_id);
555 Vend(file_id);
556 ostringstream error;
557 error << "VSattach failed for file_id=." << file_id;
558 throw_error(error.str());
559 }
560 if (FAIL == VSgetname(vid, buf)) {
561
562 Vdetach(vgroup_id);
563 Vend(file_id);
564 ostringstream error;
565 error << "VSgetname failed for file_id=." << file_id;
566 throw_error(error.str());
567 }
568 if(false == isgeo)
569 vg_dvd_map[string(buf)] = ref2;
570 else
571 vg_gvd_map[string(buf)] = ref2;
572
573 if (FAIL == VSdetach(vid)) {
574
575 Vdetach(vgroup_id);
576 Vend(file_id);
577 ostringstream error;
578 error << "VSdetach failed for file_id=." << file_id;
579 throw_error(error.str());
580
581 }
582 }// if
583 } // for
584
585 if (FAIL == Vdetach(vgroup_id)){
586 Vend(file_id);
587 ostringstream error;
588 error << "VSdetach failed for file_id=." << file_id;
589 throw_error(error.str());
590 }
591
592 return true;
593}
594
595bool HE2CF::write_attr_long_name(const string& _long_name,
596 const string& _varname,
597 int _fieldtype)
598{
599 AttrTable *at = das->get_table(_varname);
600 if (!at){
601 at = das->add_table(_varname, new AttrTable);
602 }
603 if(_fieldtype > 3){
604 at->append_attr("long_name", "String", _long_name + "(fake)");
605 }
606 else{
607 at->append_attr("long_name", "String", _long_name);
608 }
609 return true;
610}
611bool HE2CF::write_attr_long_name(const string& _group_name,
612 const string& _long_name,
613 const string& _varname,
614 int _fieldtype)
615{
616 AttrTable *at = das->get_table(_varname);
617 if (!at){
618 at = das->add_table(_varname, new AttrTable);
619 }
620 if(_fieldtype > 3){
621 at->append_attr("long_name", "String",
622 _group_name + ":" + _long_name + "(fake)");
623 }
624 else{
625 at->append_attr("long_name", "String",
626 _group_name + ":" + _long_name);
627 }
628 return true;
629}
630
631
632bool
633HE2CF::write_attr_sd(int32 _sds_id, const string& _newfname,int _fieldtype)
634{
635 char buf_var[H4_MAX_NC_NAME];
636 char buf_attr[H4_MAX_NC_NAME];
637 int32 rank;
638 int32 dimsizes[H4_MAX_VAR_DIMS];
639 int32 datatype;
640 int32 num_attrs;
641 int32 n_values;
642 intn status;
643
644 status = SDgetinfo(_sds_id, buf_var,
645 &rank, dimsizes, &datatype, &num_attrs);
646 if (FAIL == status) {
647 Vend(file_id);
648 SDendaccess(_sds_id);
649 ostringstream error;
650 error << "Cannot obtain the SDS info. ";
651 throw_error(error.str());
652
653 }
654 AttrTable *at = das->get_table(_newfname);
655 if (!at){
656 at = das->add_table(_newfname, new AttrTable);
657 }
658
659 //Check if having the "coordinates" attribute when fieldtype is 0
660 bool v_has_coordinates = false;
661 if(0 == _fieldtype) {
662 if(at->simple_find("coordinates")!= at->attr_end())
663 v_has_coordinates = true;
664 }
665
666 //Check if having the "units" attributes when fieldtype is 1(latitude) or 2(longitude)
667 bool llcv_has_units = false;
668
669 // See if we can ignore scale and offset.
670 short llcv_ignore_scale = 0;
671 short llcv_ignore_offset = 0;
672 if(1 == _fieldtype || 2 == _fieldtype) {
673 if(at->simple_find("units")!= at->attr_end())
674 llcv_has_units = true;
675 try {
676 llcv_ignore_scale = check_scale_offset(_sds_id,true);
677 }
678 catch(...) {
679 SDendaccess(_sds_id);
680 Vend(file_id);
681 throw;
682 }
683 try {
684 llcv_ignore_offset = check_scale_offset(_sds_id,false);
685 }
686 catch(...) {
687 SDendaccess(_sds_id);
688 Vend(file_id);
689 throw;
690 }
691
692 // We need to check if we ignore scale_factor and the scale_factor type is an integer.
693 // If that's the case, we need to throw an exception since we cannot retrieve the
694 // integer type scale_factor added by the HDF4 API for the latitude and longitude.
695 // We don't think such a case exists. If any, we throw an exception and hope the
696 // user to report this case to us.
697
698 // Find a case that the variable type is integer,
699 // that the scale_factor and add_offset attributes exist
700 // that the add_offset is not 0.
701 // We don't support this case and highly suspect that such a case doesn't exist.
702 if(-1 == llcv_ignore_offset && 2 == llcv_ignore_scale){
703 SDendaccess(_sds_id);
704 Vend(file_id);
705 ostringstream error;
706 error <<"The latitude or longitude has <scale_factor> and <add_offset> attributes, "
707 <<" the latitude or longitude have integer type and <add_offset> is not 0, "
708 <<" we don't support such a case in the current implementation, "
709 <<" please report to eoshelp@hdfgroup.org if you encounter this situation.";
710 throw_error(error.str());
711 }
712 }
713
714 for (int j=0; j < num_attrs; j++){
715
716 status = SDattrinfo(_sds_id, j, buf_attr, &datatype, &n_values);
717 if (status < 0){
718 Vend(file_id);
719 SDendaccess(_sds_id);
720 ostringstream error;
721 error << "SDattrinfo() failed on " << buf_attr;
722 throw_error(error.str());
723 }
724
725 if(true == v_has_coordinates) {
726 if(!strncmp(buf_attr, "coordinates", H4_MAX_NC_NAME))
727 continue;
728 }
729
730 if(true == llcv_has_units) {
731 if(!strncmp(buf_attr, "units", H4_MAX_NC_NAME))
732 continue;
733 }
734
735 if( 1 == _fieldtype || 2 == _fieldtype) {
736 if(!strncmp(buf_attr, "scale_factor", H4_MAX_NC_NAME))
737 if((1 == llcv_ignore_scale) || (2==llcv_ignore_scale) )
738 continue;
739 if(!strncmp(buf_attr, "add_offset", H4_MAX_NC_NAME))
740 if(1 == llcv_ignore_offset)
741 continue;
742 }
743
744 vector<char> value;
745 value.resize((n_values+1) * DFKNTsize(datatype));
746 status = SDreadattr(_sds_id, j, value.data());
747 if (status < 0){
748 Vend(file_id);
749 SDendaccess(_sds_id);
750 ostringstream error;
751 error << "SDreadattr() failed on " << buf_attr << endl;
752 throw_error(error.str());
753 }
754
755 // Handle character type attribute as a string.
756 if (datatype == DFNT_CHAR || datatype == DFNT_UCHAR) {
757 //*(value + n_values) = '\0';
758 value[n_values] = '\0';
759 n_values = 1;
760 }
761
762 // Need to check and change if attribute names contain special characters. KY 2012-6-8
763 string attr_cf_name = string(buf_attr,strlen(buf_attr));
764 attr_cf_name = HDFCFUtil::get_CF_string(attr_cf_name);
765 for (int loc=0; loc < n_values ; loc++) {
766 string print_rep = HDFCFUtil::print_attr(datatype, loc, (void *)value.data());
767
768 // Override any existing _FillValue attribute.
769 if (attr_cf_name == "_FillValue") {
770 at->del_attr(attr_cf_name);
771 }
772 // Override any existing long_name attribute.
773 if (attr_cf_name == "long_name") {
774 at->del_attr(attr_cf_name);
775 }
776
777 // No need to escape special characters since print_rep already does that.
778 at->append_attr(attr_cf_name, HDFCFUtil::print_type(datatype), print_rep);
779
780 }
781 }
782
783 status = SDendaccess(_sds_id);
784 if(status < 0) {
785 ostringstream error;
786 error << "SDendaccess failed on variable " << _newfname;
787 throw_error(error.str());
788 }
789
790 return true;
791}
792
793bool HE2CF::write_attr_vdata(int32 _vd_id, const string& _newfname, int _fieldtype)
794{
795 int32 number_type = 0;
796 int32 count = 0;
797 int32 size = 0;
798 char buf[H4_MAX_NC_NAME];
799
800 int vid = 0;
801
802 if ((vid = VSattach(file_id, _vd_id, "r")) < 0) {
803
804 Vend(file_id);
805
806 ostringstream error;
807 error << "VSattach failed.";
808 throw_error(error.str());
809 }
810
811 // Don't use VSnattrs - it returns the TOTAL number of attributes
812 // of a vdata and its fields. We should use VSfnattrs.
813 count = VSfnattrs(vid, _HDF_VDATA);
814 if (FAIL == count) {
815
816 VSdetach(vid);
817 Vend(file_id);
818 ostringstream error;
819 error << "VSfnattrs failed.";
820 throw_error(error.str());
821 }
822
823 AttrTable *at = das->get_table(_newfname);
824 if (!at)
825 at = das->add_table(_newfname, new AttrTable);
826
827
828 //Check if having the "coordinates" attribute when fieldtype is 0
829 bool v_has_coordinates = false;
830 if(0 == _fieldtype) {
831 if(at->simple_find("coordinates")!= at->attr_end())
832 v_has_coordinates = true;
833 }
834
835 //Check if having the "units" attributes when fieldtype is 1 or 2
836 bool llcv_has_units = false;
837 if(1 == _fieldtype || 2 == _fieldtype) {
838 if(at->simple_find("units")!= at->attr_end())
839 llcv_has_units = true;
840 }
841
842
843 for(int i=0; i < count; i++){
844 int32 count_v = 0;
845 if (VSattrinfo(vid, _HDF_VDATA, i, buf,
846 &number_type, &count_v, &size) < 0) {
847 VSdetach(vid);
848 Vend(file_id);
849 ostringstream error;
850 error << "VSattrinfo failed.";
851 throw_error(error.str());
852 }
853
854 if(true == v_has_coordinates) {
855 if(!strncmp(buf, "coordinates", H4_MAX_NC_NAME))
856 continue;
857 }
858
859 else if(true == llcv_has_units) {
860 if(!strncmp(buf, "units", H4_MAX_NC_NAME))
861 continue;
862 }
863
864 vector<char> data;
865 data.resize((count_v+1) * DFKNTsize(number_type));
866 if (VSgetattr(vid, _HDF_VDATA, i, data.data()) < 0) {
867
868 // problem: clean up and throw an exception
869 VSdetach(vid);
870 Vend(file_id);
871 ostringstream error;
872 error << "VSgetattr failed.";
873 throw_error(error.str());
874 }
875 // Handle character type attribute as a string.
876 if (number_type == DFNT_CHAR || number_type == DFNT_UCHAR8) {
877 data[count_v] ='\0';
878 count_v = 1;
879 }
880
881 for(int j=0; j < count_v ; j++){
882
883 string print_rep = HDFCFUtil::print_attr(number_type, j, (void *)data.data());
884
885 // Override any existing _FillValue attribute.
886 if(!strncmp(buf, "_FillValue", H4_MAX_NC_NAME)){
887 at->del_attr(buf);
888 }
889
890 // Override any existing long_name attribute.
891 if(!strncmp(buf, "long_name", H4_MAX_NC_NAME)){
892 at->del_attr(buf);
893 }
894
895 string vdataname(buf);
896 at->append_attr(HDFCFUtil::get_CF_string(buf), HDFCFUtil::print_type(number_type), print_rep);
897
898 }
899 } // for
900 // We need to call VFnfields to process fields as well in near future.
901 VSdetach(vid);
902
903 return true;
904}
905
906void
907HE2CF::throw_error(string _error)
908{
909 throw InternalErr(__FILE__, __LINE__,
910 _error);
911}
912
913
914// Public member functions
915HE2CF::HE2CF()
916{
917 num_global_attributes = -1;
918 file_id = -1;
919 sd_id = -1;
920 metadata = "";
921 gname = "";
922 das = nullptr;
923}
924
925HE2CF::~HE2CF()
926{
927 // Actually this is not necessary since C++ will clean up the string.
928 metadata.clear();
929}
930
931bool
933{
934
935 // Close Vgroup interface.
936 int istat = Vend(file_id);
937 if(istat == FAIL){
938 ostringstream error;
939 error << "Failed to call Vend in HE2CF::close.";
940 throw_error(error.str());
941 return false;
942 }
943
944 return true;
945}
946
947string
948HE2CF::get_metadata(const string& _name, bool& suffix_is_number,vector<string>&meta_nonnum_names, vector<string>& meta_nonum_data)
949{
950 suffix_is_number = set_metadata(_name,meta_nonnum_names,meta_nonum_data);
951 return metadata;
952}
953
954bool
955HE2CF::open(const string& _filename,const int hc_sd_id, const int hc_file_id)
956{
957 if(_filename == ""){
958 ostringstream error;
959 error << "=open(): filename is empty.";
960 throw_error(error.str());
961 return false;
962 }
963
964 if(!open_vgroup(_filename,hc_file_id)){
965 ostringstream error;
966 error << "=open(): failed to open vgroup.";
967 throw_error(error.str());
968 return false;
969 }
970
971 if(!open_sd(_filename,hc_sd_id)){
972 ostringstream error;
973 error << "=open(): failed to open sd.";
974 throw_error(error.str());
975 return false;
976 }
977
978 return true;
979}
980
981
982
983
984bool
985HE2CF::write_attribute(const string& _gname,
986 const string& _fname,
987 const string& _newfname,
988 int _n_groups,
989 int _fieldtype)
990{
991
992 if(_n_groups > 1){
993 write_attr_long_name(_gname, _fname, _newfname, _fieldtype);
994 }
995 else{
996 write_attr_long_name(_fname, _newfname, _fieldtype);
997 }
998 int32 ref_df = -1; // ref id for /Data Fields/
999 int32 ref_gf = -1; // ref id for /Geolocaion Fields/
1000
1001 // This if block effectively constructs the vg_dsd_map and vg_dvd_map at most once
1002 // since starting from the second time, gname will be equal to _gname.
1003 if(gname != _gname){
1004 // Construct field:(SD|Vdata)ref table for faster lookup.
1005 gname = _gname;
1006 get_vgroup_field_refids(_gname, &ref_df, &ref_gf);
1007 if (ref_gf !=-1)
1008 set_vgroup_map(ref_gf,true);
1009
1010 if (ref_df != 1)
1011 set_vgroup_map(ref_df,false);
1012 }
1013
1014 // Use a cached table.
1015 int32 id = -1;
1016
1017 // Using SDS name to SDS id table is fine for the HDF-EOS2 case since under one grid/swath,
1018 // the data field name is unique. Generally SDS name cannot be used as a key since one may have
1019 // the same SDS name for two different SDS objects. But in this context(HDF-EOS2 objects), it is okay.
1020 // KY 2012-7-3
1021 // Retrive sds_id for SDS
1022 // Go over all possible maps (data fields and geolocation fields)
1023 id = vg_dsd_map[_fname];
1024 if(id > 0){
1025 write_attr_sd(id, _newfname,_fieldtype);
1026 }
1027 // Retrieve ref_id for Vdata
1028 // The Vdata we retrieve here are HDF-EOS2 objects(Swath 1-D data)
1029 // Not all Vdata from the HDF file.
1030 id = vg_dvd_map[_fname];
1031 if(id > 0){
1032 write_attr_vdata(id, _newfname,_fieldtype);
1033 }
1034
1035 id = vg_gsd_map[_fname];
1036 if(id > 0){
1037 write_attr_sd(id, _newfname,_fieldtype);
1038 }
1039 // Retrieve ref_id for Vdata
1040 // The Vdata we retrieve here are HDF-EOS2 objects(Swath 1-D data)
1041 // Not all Vdata from the HDF file.
1042 id = vg_gvd_map[_fname];
1043 if(id > 0){
1044 write_attr_vdata(id, _newfname,_fieldtype);
1045 }
1046
1047 return true;
1048}
1049
1050// The application will make sure the fill value falls in the valid range of the datatype values.
1051bool
1052HE2CF::write_attribute_FillValue(const string& _varname,
1053 int _type,
1054 float value)
1055{
1056 void* v_ptr = nullptr;
1057 // Casting between pointers of different types may generate unexpected behavior.
1058 // gcc 4.8.2 reveals this problem in one test(SwathFile.HDF).
1059 // So what we should do is to use char* or vector<char> in this case and memcpy the value to the vector<char>.
1060 // Casting the vector pointer to void *. The compiler can correctly interpret the data based on the data type.
1061 vector<char>v_val;
1062
1063 // Cast the value depending on the type.
1064 switch (_type) {
1065
1066 case DFNT_UINT8:
1067 {
1068 uint8 val = (uint8) value;
1069 v_val.resize(1);
1070 memcpy(v_val.data(),&val,1);
1071 v_ptr = (void*)v_val.data();
1072 }
1073
1074 break;
1075 case DFNT_INT8:
1076 {
1077 int8 val = (int8) value;
1078 v_val.resize(1);
1079 memcpy(v_val.data(),&val,1);
1080 v_ptr = (void*)v_val.data();
1081 }
1082 break;
1083 case DFNT_INT16:
1084 {
1085 int16 val = (int16) value;
1086 v_val.resize(sizeof(short));
1087 memcpy(v_val.data(),&val,sizeof(short));
1088 v_ptr = (void*)v_val.data();
1089 }
1090 break;
1091
1092 case DFNT_UINT16:
1093 {
1094 uint16 val = (uint16) value;
1095 v_val.resize(sizeof(unsigned short));
1096 memcpy(v_val.data(),&val,sizeof(unsigned short));
1097 v_ptr = (void*)v_val.data();
1098 }
1099 break;
1100
1101 case DFNT_INT32:
1102 {
1103 int32 val = (int32) value;
1104 v_val.resize(sizeof(int));
1105 memcpy(v_val.data(),&val,sizeof(int));
1106 v_ptr = (void*)v_val.data();
1107 }
1108 break;
1109 case DFNT_UINT32:
1110 {
1111 uint32 val = (uint32) value;
1112 v_val.resize(sizeof(unsigned int));
1113 memcpy(v_val.data(),&val,sizeof(int));
1114 v_ptr = (void*)v_val.data();
1115 }
1116 break;
1117 case DFNT_FLOAT:
1118 {
1119 v_ptr = (void*)&value;
1120 }
1121 break;
1122 case DFNT_DOUBLE:
1123 {
1124 float64 val = (float64) value;
1125 v_val.resize(sizeof(double));
1126 memcpy(v_val.data(),&val,sizeof(double));
1127 v_ptr = (void*)v_val.data();
1128 }
1129 break;
1130 default:
1131 throw_error("Invalid FillValue Type - ");
1132 break;
1133 }
1134
1135 AttrTable *at = das->get_table(_varname);
1136 if (!at){
1137 at = das->add_table(_varname, new AttrTable);
1138 }
1139 string print_rep = HDFCFUtil::print_attr(_type, 0, v_ptr);
1140 at->append_attr("_FillValue", HDFCFUtil::print_type(_type), print_rep);
1141
1142 return true;
1143}
1144
1145bool
1146HE2CF::write_attribute_coordinates(const string& _varname, string _coordinates)
1147{
1148
1149 AttrTable *at = das->get_table(_varname);
1150 if (!at){
1151 at = das->add_table(_varname, new AttrTable);
1152 }
1153 at->append_attr("coordinates", "String", _coordinates);
1154
1155 return true;
1156}
1157
1158bool
1159HE2CF::write_attribute_units(const string& _varname, string _units)
1160{
1161
1162 AttrTable *at = das->get_table(_varname);
1163 if (!at){
1164 at = das->add_table(_varname, new AttrTable);
1165 }
1166 at->del_attr("units"); // Override any existing units attribute.
1167 at->append_attr("units", "String", _units);
1168
1169 return true;
1170}
1171
1172
1173// Check if the scale_factor or the add_offset can be ignored.
1174// _sds_id is the SDS ID of the variable.
1175// is_scale is true, the attribute is scale_factor.
1176// is_scale is false, the attribute is add_offset.
1177// When the attribute is scale_factor,
1178// the meaning of the returned value:
1179// 0: the scale_factor won't be ignored if exists.
1180// 1 or 2: the scale_factor can be ignored. 2 is the case when the variable type is integer. 1 is the case when the variable type is float.
1181// When the attribute is add_offset,
1182// the meaning of the returned value:
1183// 0: the add_offset won't be ignored if exists. It doesn't check if add_offset attribute exists.
1184// -1: the add_offset won't be ignored and the add_offset definitely exists,the value is not 0.
1185// 1: the add_offset will be ignored.
1186short
1187HE2CF::check_scale_offset(int32 _sds_id, bool is_scale) {
1188
1189 char buf_var[H4_MAX_NC_NAME];
1190 char buf_attr[H4_MAX_NC_NAME];
1191 int32 rank;
1192 int32 dimsizes[H4_MAX_VAR_DIMS];
1193 int32 datatype;
1194 int32 num_attrs;
1195 int32 n_values;
1196 intn status;
1197
1198 short ignore_so = 0;
1199 status = SDgetinfo(_sds_id, buf_var,
1200 &rank, dimsizes, &datatype, &num_attrs);
1201 if (FAIL == status) {
1202 SDendaccess(_sds_id);
1203 ostringstream error;
1204 error << "Cannot obtain the SDS info. ";
1205 throw_error(error.str());
1206 }
1207
1208 bool has_so = false;
1209 int so_index = -1;
1210 int32 so_dtype = -1;
1211 int32 var_dtype = datatype;
1212 string so_name =((true==is_scale)?"scale_factor":"add_offset");
1213
1214 for (int j=0; j < num_attrs; j++){
1215
1216 status = SDattrinfo(_sds_id, j, buf_attr, &datatype, &n_values);
1217 if (status < 0){
1218 SDendaccess(_sds_id);
1219 ostringstream error;
1220 error << "SDattrinfo() failed on " << buf_attr;
1221 throw_error(error.str());
1222 }
1223
1224 if(!strncmp(buf_attr, so_name.c_str(), H4_MAX_NC_NAME)) {
1225 if(1 == n_values) {
1226 has_so = true;
1227 so_index = j;
1228 so_dtype = datatype;
1229 break;
1230 }
1231 }
1232 }
1233
1234 // We find the attribute, now check if we can ignore this attribute.
1235 if(true == has_so) {
1236 vector<char> value;
1237 // We have already checked the number of attribute values. The number is 1.
1238 value.resize(DFKNTsize(so_dtype));
1239 status = SDreadattr(_sds_id, so_index, value.data());
1240 if (status < 0){
1241 SDendaccess(_sds_id);
1242 ostringstream error;
1243 error << "SDreadattr() failed on the attribute scale_factor." << endl;
1244 throw_error(error.str());
1245 }
1246 // If this attribute is "scale_factor",
1247 if(true == is_scale) {
1248 if(DFNT_FLOAT32 == so_dtype) {
1249 float final_scale_value = *((float*)((void*)(value.data())));
1250 if(final_scale_value == 1.0)
1251 ignore_so = 1;
1252 // This is the ugly case that the variable type is integer, the
1253 // scale_factor is handled by the handler reading function.
1254 // However, the add_offset for this case cannot be 0.
1255 // So we need to check this case. We may also need to
1256 // check the scale_factor value.
1257 else if(var_dtype !=DFNT_FLOAT32 && var_dtype != DFNT_FLOAT64)
1258 ignore_so = 2;
1259 }
1260 else if(DFNT_FLOAT64 == so_dtype) {
1261 double final_scale_value = *((double*)((void*)(value.data())));
1262 if(final_scale_value == 1.0)
1263 ignore_so = 1;
1264 else if(var_dtype !=DFNT_FLOAT32 && var_dtype != DFNT_FLOAT64)
1265 ignore_so = 2;
1266 }
1267 }
1268 else {
1269 // has offset, we need to make a mark for the case when scale_factor applies to a varialbe that has an integer type
1270 ignore_so = -1;
1271 string print_rep = HDFCFUtil::print_attr(so_dtype, 0, (void *)value.data());
1272 if(DFNT_FLOAT32 == so_dtype || DFNT_FLOAT64 == so_dtype) {
1273 if(atof(print_rep.c_str()) == 0.0)
1274 ignore_so = 1;
1275 }
1276 else {
1277 if(atoi(print_rep.c_str()) == 0)
1278 ignore_so = 1;
1279 }
1280 }
1281 }
1282 return ignore_so;
1283}
1284
bool open(const std::string &filename, const int sd_id, const int file_id)
openes \afilename HDF4 file.
Definition: HE2CF.cc:955
string get_metadata(const std::string &metadataname, bool &suffix_is_num, std::vector< std::string > &non_num_names, std::vector< std::string > &non_num_data)
retrieves the merged metadata.
Definition: HE2CF.cc:948
bool write_attribute(const std::string &gname, const std::string &fname, const std::string &newfname, int n_groups, int fieldtype)
Definition: HE2CF.cc:985
void set_DAS(libdap::DAS *das)
sets DAS pointer so that we can bulid attribute tables.
Definition: HE2CF.cc:181
bool close()
closes the opened file.
Definition: HE2CF.cc:932
bool write_attribute_FillValue(const std::string &varname, int type, float val)
Definition: HE2CF.cc:1052
bool write_attribute_coordinates(const std::string &varname, std::string coord)
Definition: HE2CF.cc:1146
bool write_attribute_units(const std::string &varname, std::string units)
Definition: HE2CF.cc:1159
static std::string print_attr(int32, int, void *)
Print attribute values in string.
Definition: HDFCFUtil.cc:268
static std::string print_type(int32)
Print datatype in string.
Definition: HDFCFUtil.cc:389
static std::string get_CF_string(std::string s)
Change special characters to "_".
Definition: HDFCFUtil.cc:166
static std::string escattr(std::string s)
Definition: HDFCFUtil.cc:3309